From e14f327d8094e02134efa98625acaf6fd43fee08 Mon Sep 17 00:00:00 2001
From: Liao Chunyu <chunyu@iscas.ac.cn>
Date: Tue, 17 Jun 2025 23:32:01 -0400
Subject: [RISCV] Pre-test for #144461

---
 llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
index 371ec7c790dd..522c83fd9fa9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vselect-vp.ll
@@ -470,6 +470,28 @@ define <vscale x 2 x i64> @select_nxv2i64(<vscale x 2 x i1> %a, <vscale x 2 x i6
   ret <vscale x 2 x i64> %v
 }
 
+define <vscale x 2 x i64> @select_nxv2i64_constant_true(<vscale x 2 x i1> %a, <vscale x 2 x i64> %b, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv2i64_constant_true:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmerge.vim v8, v8, -1, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i64> @llvm.vp.select.nxv2i64(<vscale x 2 x i1> %a, <vscale x 2 x i64> splat (i64 -1), <vscale x 2 x i64> %b, i32 %evl)
+  ret <vscale x 2 x i64> %v
+}
+
+define <vscale x 2 x i64> @select_nxv2i64_constant_false(<vscale x 2 x i1> %a, <vscale x 2 x i64> %b, i32 zeroext %evl) {
+; CHECK-LABEL: select_nxv2i64_constant_false:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    li a1, 100
+; CHECK-NEXT:    vsetvli zero, a0, e64, m2, ta, ma
+; CHECK-NEXT:    vmv.v.x v10, a1
+; CHECK-NEXT:    vmerge.vvm v8, v10, v8, v0
+; CHECK-NEXT:    ret
+  %v = call <vscale x 2 x i64> @llvm.vp.select.nxv2i64(<vscale x 2 x i1> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> splat (i64 100), i32 %evl)
+  ret <vscale x 2 x i64> %v
+}
+
 declare <vscale x 4 x i64> @llvm.vp.select.nxv4i64(<vscale x 4 x i1>, <vscale x 4 x i64>, <vscale x 4 x i64>, i32)
 
 define <vscale x 4 x i64> @select_nxv4i64(<vscale x 4 x i1> %a, <vscale x 4 x i64> %b, <vscale x 4 x i64> %c, i32 zeroext %evl) {
@@ -702,10 +724,10 @@ define <vscale x 16 x double> @select_nxv16f64(<vscale x 16 x i1> %a, <vscale x
 ; CHECK-NEXT:    and a4, a5, a4
 ; CHECK-NEXT:    vsetvli zero, a4, e64, m8, ta, ma
 ; CHECK-NEXT:    vmerge.vvm v16, v24, v16, v0
-; CHECK-NEXT:    bltu a2, a1, .LBB48_2
+; CHECK-NEXT:    bltu a2, a1, .LBB50_2
 ; CHECK-NEXT:  # %bb.1:
 ; CHECK-NEXT:    mv a2, a1
-; CHECK-NEXT:  .LBB48_2:
+; CHECK-NEXT:  .LBB50_2:
 ; CHECK-NEXT:    vmv1r.v v0, v7
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
-- 
cgit v1.2.3


From af49a650e172d56d684581b66afa9ab0368ec8f9 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 18 Jun 2025 13:23:17 +0900
Subject: PowerPC: Add baseline tests for more f128 libcall handling (#144381)

Some of these incorrectly call the l suffixed version of libm
functions and others assert.
---
 llvm/test/CodeGen/PowerPC/f128-arith.ll | 445 ++++++++++++++++++++++++++++++++
 1 file changed, 445 insertions(+)

diff --git a/llvm/test/CodeGen/PowerPC/f128-arith.ll b/llvm/test/CodeGen/PowerPC/f128-arith.ll
index decc4a38f7cc..ffa7ac6cb007 100644
--- a/llvm/test/CodeGen/PowerPC/f128-arith.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-arith.ll
@@ -1403,3 +1403,448 @@ entry:
   ret fp128 %3
 }
 declare { fp128, i32 } @llvm.frexp.f128.i32(fp128)
+
+
+define dso_local fp128 @acos_f128(fp128 %x) {
+; CHECK-LABEL: acos_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl acosl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: acos_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl acosl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.acos.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @asin_f128(fp128 %x) {
+; CHECK-LABEL: asin_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl asinl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: asin_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl asinl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.asin.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @atan_f128(fp128 %x) {
+; CHECK-LABEL: atan_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl atanl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: atan_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl atanl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.atan.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @atan2_f128(fp128 %x, fp128 %y) {
+; CHECK-LABEL: atan2_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl atan2l
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: atan2_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl atan2l
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.atan2.f128(fp128 %x, fp128 %y)
+  ret fp128 %result
+}
+
+define dso_local fp128 @copysign_f128(fp128 %x, fp128 %y) {
+; CHECK-LABEL: copysign_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    xscpsgnqp v2, v3, v2
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: copysign_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    xxswapd vs0, v3
+; CHECK-P8-NEXT:    addi r3, r1, -16
+; CHECK-P8-NEXT:    xxswapd vs1, v2
+; CHECK-P8-NEXT:    stxvd2x vs0, 0, r3
+; CHECK-P8-NEXT:    addi r3, r1, -32
+; CHECK-P8-NEXT:    stxvd2x vs1, 0, r3
+; CHECK-P8-NEXT:    lbz r4, -1(r1)
+; CHECK-P8-NEXT:    lbz r5, -17(r1)
+; CHECK-P8-NEXT:    rlwimi r5, r4, 0, 0, 24
+; CHECK-P8-NEXT:    stb r5, -17(r1)
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r3
+; CHECK-P8-NEXT:    xxswapd v2, vs0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.copysign.f128(fp128 %x, fp128 %y)
+  ret fp128 %result
+}
+
+define dso_local fp128 @cosh_f128(fp128 %x) {
+; CHECK-LABEL: cosh_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl coshl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: cosh_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl coshl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.cosh.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @exp10_f128(fp128 %x) {
+; CHECK-LABEL: exp10_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl exp10l
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: exp10_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl exp10l
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.exp10.f128(fp128 %x)
+  ret fp128 %result
+}
+
+; FIXME: Asserts
+; define dso_local fp128 @maximum_f128(fp128 %x, fp128 %y) {
+;   %result = call fp128 @llvm.maximum.f128(fp128 %x, fp128 %y)
+;   ret fp128 %result
+; }
+
+; FIXME: Asserts
+; define dso_local fp128 @minimum_f128(fp128 %x, fp128 %y) {
+;   %result = call fp128 @llvm.minimum.f128(fp128 %x, fp128 %y)
+;   ret fp128 %result
+; }
+
+; FIXME: Asserts
+; define dso_local fp128 @maximumnum_f128(fp128 %x, fp128 %y) {
+;   %result = call fp128 @llvm.maximumnum.f128(fp128 %x, fp128 %y)
+;   ret fp128 %result
+; }
+
+; FIXME: Asserts
+; define dso_local fp128 @minimumnum_f128(fp128 %x, fp128 %y) {
+;   %result = call fp128 @llvm.minimumnum.f128(fp128 %x, fp128 %y)
+;   ret fp128 %result
+; }
+
+define dso_local fp128 @ldexp_f128(fp128 %x, i32 %y) {
+; CHECK-LABEL: ldexp_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    extsw r5, r5
+; CHECK-NEXT:    bl ldexpl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: ldexp_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    extsw r5, r5
+; CHECK-P8-NEXT:    bl ldexpl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.ldexp.f128.i32(fp128 %x, i32 %y)
+  ret fp128 %result
+}
+
+define dso_local { fp128, fp128 } @modf_f128(fp128 %x) {
+; CHECK-LABEL: modf_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -48(r1)
+; CHECK-NEXT:    std r0, 64(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 48
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    addi r5, r1, 32
+; CHECK-NEXT:    bl modfl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    lxv v3, 32(r1)
+; CHECK-NEXT:    addi r1, r1, 48
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: modf_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 64
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    .cfi_offset r30, -16
+; CHECK-P8-NEXT:    std r30, -16(r1) # 8-byte Folded Spill
+; CHECK-P8-NEXT:    stdu r1, -64(r1)
+; CHECK-P8-NEXT:    addi r30, r1, 32
+; CHECK-P8-NEXT:    std r0, 80(r1)
+; CHECK-P8-NEXT:    mr r5, r30
+; CHECK-P8-NEXT:    bl modfl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    lxvd2x vs0, 0, r30
+; CHECK-P8-NEXT:    xxswapd v3, vs0
+; CHECK-P8-NEXT:    addi r1, r1, 64
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    ld r30, -16(r1) # 8-byte Folded Reload
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call { fp128, fp128 } @llvm.modf.f128(fp128 %x)
+  ret { fp128, fp128 } %result
+}
+
+define dso_local fp128 @roundeven_f128(fp128 %x) {
+; CHECK-LABEL: roundeven_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl roundevenl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: roundeven_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl roundevenl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.roundeven.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @sinh_f128(fp128 %x) {
+; CHECK-LABEL: sinh_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl sinhl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: sinh_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl sinhl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.sinh.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @tanh_f128(fp128 %x) {
+; CHECK-LABEL: tanh_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl tanhl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: tanh_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl tanhl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.tanh.f128(fp128 %x)
+  ret fp128 %result
+}
+
+define dso_local fp128 @tan_f128(fp128 %x) {
+; CHECK-LABEL: tan_f128:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    stdu r1, -32(r1)
+; CHECK-NEXT:    std r0, 48(r1)
+; CHECK-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NEXT:    .cfi_offset lr, 16
+; CHECK-NEXT:    bl tanl
+; CHECK-NEXT:    nop
+; CHECK-NEXT:    addi r1, r1, 32
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; CHECK-P8-LABEL: tan_f128:
+; CHECK-P8:       # %bb.0:
+; CHECK-P8-NEXT:    mflr r0
+; CHECK-P8-NEXT:    stdu r1, -32(r1)
+; CHECK-P8-NEXT:    std r0, 48(r1)
+; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-P8-NEXT:    .cfi_offset lr, 16
+; CHECK-P8-NEXT:    bl tanl
+; CHECK-P8-NEXT:    nop
+; CHECK-P8-NEXT:    addi r1, r1, 32
+; CHECK-P8-NEXT:    ld r0, 16(r1)
+; CHECK-P8-NEXT:    mtlr r0
+; CHECK-P8-NEXT:    blr
+  %result = call fp128 @llvm.tan.f128(fp128 %x)
+  ret fp128 %result
+}
-- 
cgit v1.2.3


From 7b9d10d2e6410029fd0750b2e0566432dbf03dc7 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 18 Jun 2025 13:26:15 +0900
Subject: PowerPC: Fix using long double libm functions for f128 intrinsics
 (#144382)

This wasn't setting the correct libcall names, which default to the
l suffixed libm names.
---
 llvm/lib/IR/RuntimeLibcalls.cpp         | 143 +++++++++++++++-----------------
 llvm/test/CodeGen/PowerPC/f128-arith.ll |  48 +++++------
 2 files changed, 91 insertions(+), 100 deletions(-)

diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 7396626a03d4..a57b08919346 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -281,6 +281,69 @@ void RuntimeLibcallsInfo::initSoftFloatCmpLibcallPredicates() {
   SoftFloatCompareLibcallPredicates[RTLIB::UO_PPCF128] = CmpInst::ICMP_NE;
 }
 
+static void setLongDoubleIsF128Libm(RuntimeLibcallsInfo &Info,
+                                    bool FiniteOnlyFuncs = false) {
+  Info.setLibcallName(RTLIB::REM_F128, "fmodf128");
+  Info.setLibcallName(RTLIB::FMA_F128, "fmaf128");
+  Info.setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
+  Info.setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
+  Info.setLibcallName(RTLIB::LOG_F128, "logf128");
+  Info.setLibcallName(RTLIB::LOG2_F128, "log2f128");
+  Info.setLibcallName(RTLIB::LOG10_F128, "log10f128");
+  Info.setLibcallName(RTLIB::EXP_F128, "expf128");
+  Info.setLibcallName(RTLIB::EXP2_F128, "exp2f128");
+  Info.setLibcallName(RTLIB::EXP10_F128, "exp10f128");
+  Info.setLibcallName(RTLIB::SIN_F128, "sinf128");
+  Info.setLibcallName(RTLIB::COS_F128, "cosf128");
+  Info.setLibcallName(RTLIB::TAN_F128, "tanf128");
+  Info.setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
+  Info.setLibcallName(RTLIB::ASIN_F128, "asinf128");
+  Info.setLibcallName(RTLIB::ACOS_F128, "acosf128");
+  Info.setLibcallName(RTLIB::ATAN_F128, "atanf128");
+  Info.setLibcallName(RTLIB::ATAN2_F128, "atan2f128");
+  Info.setLibcallName(RTLIB::SINH_F128, "sinhf128");
+  Info.setLibcallName(RTLIB::COSH_F128, "coshf128");
+  Info.setLibcallName(RTLIB::TANH_F128, "tanhf128");
+  Info.setLibcallName(RTLIB::POW_F128, "powf128");
+  Info.setLibcallName(RTLIB::CEIL_F128, "ceilf128");
+  Info.setLibcallName(RTLIB::TRUNC_F128, "truncf128");
+  Info.setLibcallName(RTLIB::RINT_F128, "rintf128");
+  Info.setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
+  Info.setLibcallName(RTLIB::ROUND_F128, "roundf128");
+  Info.setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
+  Info.setLibcallName(RTLIB::FLOOR_F128, "floorf128");
+  Info.setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
+  Info.setLibcallName(RTLIB::FMIN_F128, "fminf128");
+  Info.setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
+  Info.setLibcallName(RTLIB::FMINIMUM_F128, "fminimumf128");
+  Info.setLibcallName(RTLIB::FMAXIMUM_F128, "fmaximumf128");
+  Info.setLibcallName(RTLIB::FMINIMUM_NUM_F128, "fminimum_numf128");
+  Info.setLibcallName(RTLIB::FMAXIMUM_NUM_F128, "fmaximum_numf128");
+  Info.setLibcallName(RTLIB::LROUND_F128, "lroundf128");
+  Info.setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
+  Info.setLibcallName(RTLIB::LRINT_F128, "lrintf128");
+  Info.setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
+  Info.setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
+  Info.setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+  Info.setLibcallName(RTLIB::MODF_F128, "modff128");
+
+  if (FiniteOnlyFuncs) {
+    Info.setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
+    Info.setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
+    Info.setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
+    Info.setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
+    Info.setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
+    Info.setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
+  } else {
+    Info.setLibcallName(RTLIB::LOG_FINITE_F128, nullptr);
+    Info.setLibcallName(RTLIB::LOG2_FINITE_F128, nullptr);
+    Info.setLibcallName(RTLIB::LOG10_FINITE_F128, nullptr);
+    Info.setLibcallName(RTLIB::EXP_FINITE_F128, nullptr);
+    Info.setLibcallName(RTLIB::EXP2_FINITE_F128, nullptr);
+    Info.setLibcallName(RTLIB::POW_FINITE_F128, nullptr);
+  }
+}
+
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
 void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
@@ -295,57 +358,8 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
 #undef LIBCALL_NO_NAME
 
   // Use the f128 variants of math functions on x86
-  if (TT.isX86() && TT.isGNUEnvironment()) {
-    setLibcallName(RTLIB::REM_F128, "fmodf128");
-    setLibcallName(RTLIB::FMA_F128, "fmaf128");
-    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-    setLibcallName(RTLIB::CBRT_F128, "cbrtf128");
-    setLibcallName(RTLIB::LOG_F128, "logf128");
-    setLibcallName(RTLIB::LOG_FINITE_F128, "__logf128_finite");
-    setLibcallName(RTLIB::LOG2_F128, "log2f128");
-    setLibcallName(RTLIB::LOG2_FINITE_F128, "__log2f128_finite");
-    setLibcallName(RTLIB::LOG10_F128, "log10f128");
-    setLibcallName(RTLIB::LOG10_FINITE_F128, "__log10f128_finite");
-    setLibcallName(RTLIB::EXP_F128, "expf128");
-    setLibcallName(RTLIB::EXP_FINITE_F128, "__expf128_finite");
-    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-    setLibcallName(RTLIB::EXP2_FINITE_F128, "__exp2f128_finite");
-    setLibcallName(RTLIB::EXP10_F128, "exp10f128");
-    setLibcallName(RTLIB::SIN_F128, "sinf128");
-    setLibcallName(RTLIB::COS_F128, "cosf128");
-    setLibcallName(RTLIB::TAN_F128, "tanf128");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-    setLibcallName(RTLIB::ASIN_F128, "asinf128");
-    setLibcallName(RTLIB::ACOS_F128, "acosf128");
-    setLibcallName(RTLIB::ATAN_F128, "atanf128");
-    setLibcallName(RTLIB::ATAN2_F128, "atan2f128");
-    setLibcallName(RTLIB::SINH_F128, "sinhf128");
-    setLibcallName(RTLIB::COSH_F128, "coshf128");
-    setLibcallName(RTLIB::TANH_F128, "tanhf128");
-    setLibcallName(RTLIB::POW_F128, "powf128");
-    setLibcallName(RTLIB::POW_FINITE_F128, "__powf128_finite");
-    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-    setLibcallName(RTLIB::RINT_F128, "rintf128");
-    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-    setLibcallName(RTLIB::ROUND_F128, "roundf128");
-    setLibcallName(RTLIB::ROUNDEVEN_F128, "roundevenf128");
-    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-    setLibcallName(RTLIB::COPYSIGN_F128, "copysignf128");
-    setLibcallName(RTLIB::FMIN_F128, "fminf128");
-    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-    setLibcallName(RTLIB::FMINIMUM_F128, "fminimumf128");
-    setLibcallName(RTLIB::FMAXIMUM_F128, "fmaximumf128");
-    setLibcallName(RTLIB::FMINIMUM_NUM_F128, "fminimum_numf128");
-    setLibcallName(RTLIB::FMAXIMUM_NUM_F128, "fmaximum_numf128");
-    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-    setLibcallName(RTLIB::LDEXP_F128, "ldexpf128");
-    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
-    setLibcallName(RTLIB::MODF_F128, "modff128");
-  }
+  if (TT.isX86() && TT.isGNUEnvironment())
+    setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/true);
 
   // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
   if (TT.isPPC()) {
@@ -379,31 +393,8 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
     setLibcallName(RTLIB::OGT_F128, "__gtkf2");
     setLibcallName(RTLIB::UO_F128, "__unordkf2");
 
-    setLibcallName(RTLIB::LOG_F128, "logf128");
-    setLibcallName(RTLIB::LOG2_F128, "log2f128");
-    setLibcallName(RTLIB::LOG10_F128, "log10f128");
-    setLibcallName(RTLIB::EXP_F128, "expf128");
-    setLibcallName(RTLIB::EXP2_F128, "exp2f128");
-    setLibcallName(RTLIB::SIN_F128, "sinf128");
-    setLibcallName(RTLIB::COS_F128, "cosf128");
-    setLibcallName(RTLIB::SINCOS_F128, "sincosf128");
-    setLibcallName(RTLIB::POW_F128, "powf128");
-    setLibcallName(RTLIB::FMIN_F128, "fminf128");
-    setLibcallName(RTLIB::FMAX_F128, "fmaxf128");
-    setLibcallName(RTLIB::REM_F128, "fmodf128");
-    setLibcallName(RTLIB::SQRT_F128, "sqrtf128");
-    setLibcallName(RTLIB::CEIL_F128, "ceilf128");
-    setLibcallName(RTLIB::FLOOR_F128, "floorf128");
-    setLibcallName(RTLIB::TRUNC_F128, "truncf128");
-    setLibcallName(RTLIB::ROUND_F128, "roundf128");
-    setLibcallName(RTLIB::LROUND_F128, "lroundf128");
-    setLibcallName(RTLIB::LLROUND_F128, "llroundf128");
-    setLibcallName(RTLIB::RINT_F128, "rintf128");
-    setLibcallName(RTLIB::LRINT_F128, "lrintf128");
-    setLibcallName(RTLIB::LLRINT_F128, "llrintf128");
-    setLibcallName(RTLIB::NEARBYINT_F128, "nearbyintf128");
-    setLibcallName(RTLIB::FMA_F128, "fmaf128");
-    setLibcallName(RTLIB::FREXP_F128, "frexpf128");
+    // TODO: Do the finite only functions exist?
+    setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/false);
 
     if (TT.isOSAIX()) {
       bool isPPC64 = TT.isPPC64();
diff --git a/llvm/test/CodeGen/PowerPC/f128-arith.ll b/llvm/test/CodeGen/PowerPC/f128-arith.ll
index ffa7ac6cb007..f9c953d483ff 100644
--- a/llvm/test/CodeGen/PowerPC/f128-arith.ll
+++ b/llvm/test/CodeGen/PowerPC/f128-arith.ll
@@ -1413,7 +1413,7 @@ define dso_local fp128 @acos_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl acosl
+; CHECK-NEXT:    bl acosf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1427,7 +1427,7 @@ define dso_local fp128 @acos_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl acosl
+; CHECK-P8-NEXT:    bl acosf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1445,7 +1445,7 @@ define dso_local fp128 @asin_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl asinl
+; CHECK-NEXT:    bl asinf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1459,7 +1459,7 @@ define dso_local fp128 @asin_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl asinl
+; CHECK-P8-NEXT:    bl asinf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1477,7 +1477,7 @@ define dso_local fp128 @atan_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl atanl
+; CHECK-NEXT:    bl atanf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1491,7 +1491,7 @@ define dso_local fp128 @atan_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl atanl
+; CHECK-P8-NEXT:    bl atanf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1509,7 +1509,7 @@ define dso_local fp128 @atan2_f128(fp128 %x, fp128 %y) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl atan2l
+; CHECK-NEXT:    bl atan2f128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1523,7 +1523,7 @@ define dso_local fp128 @atan2_f128(fp128 %x, fp128 %y) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl atan2l
+; CHECK-P8-NEXT:    bl atan2f128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1566,7 +1566,7 @@ define dso_local fp128 @cosh_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl coshl
+; CHECK-NEXT:    bl coshf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1580,7 +1580,7 @@ define dso_local fp128 @cosh_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl coshl
+; CHECK-P8-NEXT:    bl coshf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1598,7 +1598,7 @@ define dso_local fp128 @exp10_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl exp10l
+; CHECK-NEXT:    bl exp10f128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1612,7 +1612,7 @@ define dso_local fp128 @exp10_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl exp10l
+; CHECK-P8-NEXT:    bl exp10f128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1655,7 +1655,7 @@ define dso_local fp128 @ldexp_f128(fp128 %x, i32 %y) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    extsw r5, r5
-; CHECK-NEXT:    bl ldexpl
+; CHECK-NEXT:    bl ldexpf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1670,7 +1670,7 @@ define dso_local fp128 @ldexp_f128(fp128 %x, i32 %y) {
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
 ; CHECK-P8-NEXT:    extsw r5, r5
-; CHECK-P8-NEXT:    bl ldexpl
+; CHECK-P8-NEXT:    bl ldexpf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1689,7 +1689,7 @@ define dso_local { fp128, fp128 } @modf_f128(fp128 %x) {
 ; CHECK-NEXT:    .cfi_def_cfa_offset 48
 ; CHECK-NEXT:    .cfi_offset lr, 16
 ; CHECK-NEXT:    addi r5, r1, 32
-; CHECK-NEXT:    bl modfl
+; CHECK-NEXT:    bl modff128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    lxv v3, 32(r1)
 ; CHECK-NEXT:    addi r1, r1, 48
@@ -1708,7 +1708,7 @@ define dso_local { fp128, fp128 } @modf_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    addi r30, r1, 32
 ; CHECK-P8-NEXT:    std r0, 80(r1)
 ; CHECK-P8-NEXT:    mr r5, r30
-; CHECK-P8-NEXT:    bl modfl
+; CHECK-P8-NEXT:    bl modff128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    lxvd2x vs0, 0, r30
 ; CHECK-P8-NEXT:    xxswapd v3, vs0
@@ -1729,7 +1729,7 @@ define dso_local fp128 @roundeven_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl roundevenl
+; CHECK-NEXT:    bl roundevenf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1743,7 +1743,7 @@ define dso_local fp128 @roundeven_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl roundevenl
+; CHECK-P8-NEXT:    bl roundevenf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1761,7 +1761,7 @@ define dso_local fp128 @sinh_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl sinhl
+; CHECK-NEXT:    bl sinhf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1775,7 +1775,7 @@ define dso_local fp128 @sinh_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl sinhl
+; CHECK-P8-NEXT:    bl sinhf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1793,7 +1793,7 @@ define dso_local fp128 @tanh_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl tanhl
+; CHECK-NEXT:    bl tanhf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1807,7 +1807,7 @@ define dso_local fp128 @tanh_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl tanhl
+; CHECK-P8-NEXT:    bl tanhf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
@@ -1825,7 +1825,7 @@ define dso_local fp128 @tan_f128(fp128 %x) {
 ; CHECK-NEXT:    std r0, 48(r1)
 ; CHECK-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-NEXT:    .cfi_offset lr, 16
-; CHECK-NEXT:    bl tanl
+; CHECK-NEXT:    bl tanf128
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    addi r1, r1, 32
 ; CHECK-NEXT:    ld r0, 16(r1)
@@ -1839,7 +1839,7 @@ define dso_local fp128 @tan_f128(fp128 %x) {
 ; CHECK-P8-NEXT:    std r0, 48(r1)
 ; CHECK-P8-NEXT:    .cfi_def_cfa_offset 32
 ; CHECK-P8-NEXT:    .cfi_offset lr, 16
-; CHECK-P8-NEXT:    bl tanl
+; CHECK-P8-NEXT:    bl tanf128
 ; CHECK-P8-NEXT:    nop
 ; CHECK-P8-NEXT:    addi r1, r1, 32
 ; CHECK-P8-NEXT:    ld r0, 16(r1)
-- 
cgit v1.2.3


From ad9e591fd53f2cf91a2744973b59669d873658af Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Tue, 17 Jun 2025 21:33:50 -0700
Subject: [SelectionDAG][RISCV] Fold (add (vscale * C0), (vscale * C1)) to
 (vscale * (C0 + C1)) in getNode. (#144565)

We already have shl/mul vscale related folds in getNode.

This is an alternative to the DAGCombine proposed in #144507.
---
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp     |   7 +
 llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll   |   4 -
 llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll    |  32 +-
 llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll         | 283 +++----
 llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll            |  12 +-
 llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll        |   9 +-
 llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll       |   3 +-
 .../CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll  |   2 +-
 .../CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll | 184 ++---
 llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll |  18 -
 .../CodeGen/RISCV/rvv/vector-interleave-store.ll   |   9 +-
 llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll   | 817 ++++++++++-----------
 llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll       |  12 +-
 .../RISCV/rvv/vp-vector-interleaved-access.ll      | 100 ++-
 14 files changed, 682 insertions(+), 810 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 45a37622a531..b0e3f534e2aa 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -7377,6 +7377,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
     if ((Opcode == ISD::ADD || Opcode == ISD::SUB) &&
         VT.getScalarType() == MVT::i1)
       return getNode(ISD::XOR, DL, VT, N1, N2);
+    // Fold (add (vscale * C0), (vscale * C1)) to (vscale * (C0 + C1)).
+    if (Opcode == ISD::ADD && N1.getOpcode() == ISD::VSCALE &&
+        N2.getOpcode() == ISD::VSCALE) {
+      const APInt &C1 = N1->getConstantOperandAPInt(0);
+      const APInt &C2 = N2->getConstantOperandAPInt(0);
+      return getVScale(DL, VT, C1 + C2);
+    }
     break;
   case ISD::MUL:
     assert(VT.isInteger() && "This operator does not apply to FP types!");
diff --git a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
index d42c42c7ce03..7c9a283dd54b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/extract-subvector.ll
@@ -488,8 +488,6 @@ define <vscale x 6 x half> @extract_nxv6f16_nxv12f16_6(<vscale x 12 x half> %in)
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v13, v10, a0
 ; CHECK-NEXT:    vslidedown.vx v12, v9, a0
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v12, v10, a0
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
@@ -543,8 +541,6 @@ define <vscale x 6 x bfloat> @extract_nxv6bf16_nxv12bf16_6(<vscale x 12 x bfloat
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v13, v10, a0
 ; CHECK-NEXT:    vslidedown.vx v12, v9, a0
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v12, v10, a0
 ; CHECK-NEXT:    vmv2r.v v8, v12
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
index 61cf1f56aee3..22ddd4f8a95d 100644
--- a/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/insert-subvector.ll
@@ -81,8 +81,7 @@ define <vscale x 4 x i8> @insert_nxv1i8_nxv4i8_3(<vscale x 4 x i8> %vec, <vscale
 ; CHECK-NEXT:    srli a1, a0, 3
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 4 x i8> @llvm.vector.insert.nxv1i8.nxv4i8(<vscale x 4 x i8> %vec, <vscale x 1 x i8> %subvec, i64 3)
@@ -246,8 +245,7 @@ define <vscale x 16 x i32> @insert_nxv16i32_nxv1i32_1(<vscale x 16 x i32> %vec,
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v16, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i32> @llvm.vector.insert.nxv1i32.nxv16i32(<vscale x 16 x i32> %vec, <vscale x 1 x i32> %subvec, i64 1)
@@ -282,8 +280,8 @@ define <vscale x 16 x i8> @insert_nxv16i8_nxv1i8_1(<vscale x 16 x i8> %vec, <vsc
 ; CHECK-LABEL: insert_nxv16i8_nxv1i8_1:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
 ; CHECK-NEXT:    ret
@@ -310,11 +308,11 @@ define <vscale x 16 x i8> @insert_nxv16i8_nxv1i8_3(<vscale x 16 x i8> %vec, <vsc
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a0, a0, a1
-; CHECK-NEXT:    add a1, a0, a1
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, tu, ma
-; CHECK-NEXT:    vslideup.vx v8, v10, a0
+; CHECK-NEXT:    srli a2, a0, 2
+; CHECK-NEXT:    add a1, a2, a1
+; CHECK-NEXT:    srli a0, a0, 1
+; CHECK-NEXT:    vsetvli zero, a0, e8, m1, tu, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    ret
   %v = call <vscale x 16 x i8> @llvm.vector.insert.nxv1i8.nxv16i8(<vscale x 16 x i8> %vec, <vscale x 1 x i8> %subvec, i64 3)
   ret <vscale x 16 x i8> %v
@@ -363,8 +361,7 @@ define <vscale x 32 x half> @insert_nxv32f16_nxv2f16_2(<vscale x 32 x half> %vec
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v16, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 32 x half> @llvm.vector.insert.nxv2f16.nxv32f16(<vscale x 32 x half> %vec, <vscale x 2 x half> %subvec, i64 2)
@@ -376,8 +373,7 @@ define <vscale x 32 x half> @insert_nxv32f16_nxv2f16_26(<vscale x 32 x half> %ve
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v14, v16, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 32 x half> @llvm.vector.insert.nxv2f16.nxv32f16(<vscale x 32 x half> %vec, <vscale x 2 x half> %subvec, i64 26)
@@ -422,8 +418,8 @@ define <vscale x 32 x i1> @insert_nxv32i1_nxv8i1_8(<vscale x 32 x i1> %v, <vscal
 ; CHECK-LABEL: insert_nxv32i1_nxv8i1_8:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
 ; CHECK-NEXT:    vslideup.vx v0, v8, a0
 ; CHECK-NEXT:    ret
@@ -570,8 +566,7 @@ define <vscale x 32 x bfloat> @insert_nxv32bf16_nxv2bf16_2(<vscale x 32 x bfloat
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v16, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv2bf16.nxv32bf16(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec, i64 2)
@@ -583,8 +578,7 @@ define <vscale x 32 x bfloat> @insert_nxv32bf16_nxv2bf16_26(<vscale x 32 x bfloa
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v14, v16, a0
 ; CHECK-NEXT:    ret
   %v = call <vscale x 32 x bfloat> @llvm.vector.insert.nxv2bf16.nxv32bf16(<vscale x 32 x bfloat> %vec, <vscale x 2 x bfloat> %subvec, i64 26)
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
index 28b27bb75f21..9972df97ad9f 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp-vp.ll
@@ -1371,6 +1371,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    slli a1, a1, 1
 ; CHECK-NEXT:    mv a3, a1
 ; CHECK-NEXT:    slli a1, a1, 2
+; CHECK-NEXT:    add a3, a3, a1
+; CHECK-NEXT:    slli a1, a1, 1
 ; CHECK-NEXT:    add a1, a1, a3
 ; CHECK-NEXT:    add a1, sp, a1
 ; CHECK-NEXT:    addi a1, a1, 16
@@ -1378,9 +1380,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vmv8r.v v0, v16
 ; CHECK-NEXT:    csrr a1, vlenb
-; CHECK-NEXT:    mv a3, a1
 ; CHECK-NEXT:    slli a1, a1, 1
-; CHECK-NEXT:    add a3, a3, a1
+; CHECK-NEXT:    mv a3, a1
 ; CHECK-NEXT:    slli a1, a1, 3
 ; CHECK-NEXT:    add a1, a1, a3
 ; CHECK-NEXT:    add a1, sp, a1
@@ -1406,6 +1407,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    slli t0, t0, 1
 ; CHECK-NEXT:    mv t1, t0
 ; CHECK-NEXT:    slli t0, t0, 2
+; CHECK-NEXT:    add t1, t1, t0
+; CHECK-NEXT:    slli t0, t0, 1
 ; CHECK-NEXT:    add t0, t0, t1
 ; CHECK-NEXT:    add t0, sp, t0
 ; CHECK-NEXT:    addi t0, t0, 16
@@ -1413,9 +1416,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    vslidedown.vx v16, v8, a1
 ; CHECK-NEXT:    vl8re16.v v8, (a0)
 ; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    mv t0, a0
 ; CHECK-NEXT:    slli a0, a0, 1
-; CHECK-NEXT:    add t0, t0, a0
+; CHECK-NEXT:    mv t0, a0
 ; CHECK-NEXT:    slli a0, a0, 2
 ; CHECK-NEXT:    add a0, a0, t0
 ; CHECK-NEXT:    add a0, sp, a0
@@ -1445,10 +1447,6 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v5, v8, v16, v0.t
-; CHECK-NEXT:    csrr a0, vlenb
-; CHECK-NEXT:    add a0, sp, a0
-; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, a6, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v24
 ; CHECK-NEXT:    csrr a0, vlenb
@@ -1457,85 +1455,95 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64bf16(<vscale x 64 x bfloat> %va, <vs
 ; CHECK-NEXT:    addi a0, a0, 16
 ; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmfeq.vv v6, v24, v16, v0.t
-; CHECK-NEXT:    add a0, a3, a3
+; CHECK-NEXT:    vmfeq.vv v7, v24, v16, v0.t
 ; CHECK-NEXT:    bltu a2, a5, .LBB85_4
 ; CHECK-NEXT:  # %bb.3:
 ; CHECK-NEXT:    mv a2, a5
 ; CHECK-NEXT:  .LBB85_4:
-; CHECK-NEXT:    sub a5, a2, a4
-; CHECK-NEXT:    csrr a6, vlenb
-; CHECK-NEXT:    slli a6, a6, 1
-; CHECK-NEXT:    mv a7, a6
-; CHECK-NEXT:    slli a6, a6, 2
-; CHECK-NEXT:    add a6, a6, a7
-; CHECK-NEXT:    add a6, sp, a6
-; CHECK-NEXT:    addi a6, a6, 16
-; CHECK-NEXT:    vl1r.v v7, (a6) # vscale x 8-byte Folded Reload
-; CHECK-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vx v0, v7, a3
-; CHECK-NEXT:    sltu a6, a2, a5
-; CHECK-NEXT:    addi a6, a6, -1
-; CHECK-NEXT:    and a5, a6, a5
-; CHECK-NEXT:    csrr a6, vlenb
-; CHECK-NEXT:    mv a7, a6
-; CHECK-NEXT:    slli a6, a6, 1
-; CHECK-NEXT:    add a7, a7, a6
-; CHECK-NEXT:    slli a6, a6, 3
-; CHECK-NEXT:    add a6, a6, a7
-; CHECK-NEXT:    add a6, sp, a6
-; CHECK-NEXT:    addi a6, a6, 16
-; CHECK-NEXT:    vl8r.v v24, (a6) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v28
+; CHECK-NEXT:    sub a0, a2, a4
 ; CHECK-NEXT:    csrr a5, vlenb
-; CHECK-NEXT:    mv a6, a5
 ; CHECK-NEXT:    slli a5, a5, 1
-; CHECK-NEXT:    add a6, a6, a5
+; CHECK-NEXT:    mv a6, a5
 ; CHECK-NEXT:    slli a5, a5, 2
+; CHECK-NEXT:    add a6, a6, a5
+; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    add a5, a5, a6
+; CHECK-NEXT:    add a5, sp, a5
+; CHECK-NEXT:    addi a5, a5, 16
+; CHECK-NEXT:    vl1r.v v8, (a5) # vscale x 8-byte Folded Reload
+; CHECK-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslidedown.vx v0, v8, a3
+; CHECK-NEXT:    sltu a5, a2, a0
+; CHECK-NEXT:    addi a5, a5, -1
+; CHECK-NEXT:    and a0, a5, a0
+; CHECK-NEXT:    csrr a5, vlenb
+; CHECK-NEXT:    slli a5, a5, 1
+; CHECK-NEXT:    mv a6, a5
+; CHECK-NEXT:    slli a5, a5, 3
 ; CHECK-NEXT:    add a5, a5, a6
 ; CHECK-NEXT:    add a5, sp, a5
 ; CHECK-NEXT:    addi a5, a5, 16
-; CHECK-NEXT:    vl8r.v v8, (a5) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vl8r.v v24, (a5) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v28
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    mv a5, a0
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add a0, a0, a5
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v12
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vmfeq.vv v4, v16, v24, v0.t
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v6, v5, a3
+; CHECK-NEXT:    vmfeq.vv v10, v16, v24, v0.t
+; CHECK-NEXT:    vmv1r.v v9, v7
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v9, v5, a3
 ; CHECK-NEXT:    bltu a2, a4, .LBB85_6
 ; CHECK-NEXT:  # %bb.5:
 ; CHECK-NEXT:    mv a2, a4
 ; CHECK-NEXT:  .LBB85_6:
-; CHECK-NEXT:    vmv1r.v v0, v7
-; CHECK-NEXT:    csrr a4, vlenb
-; CHECK-NEXT:    mv a5, a4
-; CHECK-NEXT:    slli a4, a4, 1
-; CHECK-NEXT:    add a5, a5, a4
-; CHECK-NEXT:    slli a4, a4, 3
-; CHECK-NEXT:    add a4, a4, a5
-; CHECK-NEXT:    add a4, sp, a4
-; CHECK-NEXT:    addi a4, a4, 16
-; CHECK-NEXT:    vl8r.v v24, (a4) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    mv a4, a0
+; CHECK-NEXT:    slli a0, a0, 3
+; CHECK-NEXT:    add a0, a0, a4
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
 ; CHECK-NEXT:    vfwcvtbf16.f.f.v v16, v24
-; CHECK-NEXT:    csrr a2, vlenb
-; CHECK-NEXT:    mv a4, a2
-; CHECK-NEXT:    slli a2, a2, 1
-; CHECK-NEXT:    add a4, a4, a2
-; CHECK-NEXT:    slli a2, a2, 2
-; CHECK-NEXT:    add a2, a2, a4
-; CHECK-NEXT:    add a2, sp, a2
-; CHECK-NEXT:    addi a2, a2, 16
-; CHECK-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
-; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v8
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    mv a2, a0
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl8r.v v0, (a0) # vscale x 64-byte Folded Reload
+; CHECK-NEXT:    vfwcvtbf16.f.f.v v24, v0
+; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    mv a2, a0
+; CHECK-NEXT:    slli a0, a0, 2
+; CHECK-NEXT:    add a2, a2, a0
+; CHECK-NEXT:    slli a0, a0, 1
+; CHECK-NEXT:    add a0, a0, a2
+; CHECK-NEXT:    add a0, sp, a0
+; CHECK-NEXT:    addi a0, a0, 16
+; CHECK-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; CHECK-NEXT:    vmfeq.vv v8, v16, v24, v0.t
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v4, a3
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v6, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v10, a3
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v9, a1
 ; CHECK-NEXT:    vmv.v.v v0, v8
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    mv a1, a0
@@ -3546,8 +3554,7 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFH-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; ZVFH-NEXT:    vsetvli zero, a2, e16, m8, ta, ma
 ; ZVFH-NEXT:    vmfeq.vv v16, v24, v8, v0.t
-; ZVFH-NEXT:    add a0, a1, a1
-; ZVFH-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
+; ZVFH-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
 ; ZVFH-NEXT:    vslideup.vx v16, v6, a1
 ; ZVFH-NEXT:    vmv.v.v v0, v16
 ; ZVFH-NEXT:    csrr a0, vlenb
@@ -3576,6 +3583,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    slli a1, a1, 1
 ; ZVFHMIN-NEXT:    mv a3, a1
 ; ZVFHMIN-NEXT:    slli a1, a1, 2
+; ZVFHMIN-NEXT:    add a3, a3, a1
+; ZVFHMIN-NEXT:    slli a1, a1, 1
 ; ZVFHMIN-NEXT:    add a1, a1, a3
 ; ZVFHMIN-NEXT:    add a1, sp, a1
 ; ZVFHMIN-NEXT:    addi a1, a1, 16
@@ -3583,9 +3592,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; ZVFHMIN-NEXT:    vmv8r.v v0, v16
 ; ZVFHMIN-NEXT:    csrr a1, vlenb
-; ZVFHMIN-NEXT:    mv a3, a1
 ; ZVFHMIN-NEXT:    slli a1, a1, 1
-; ZVFHMIN-NEXT:    add a3, a3, a1
+; ZVFHMIN-NEXT:    mv a3, a1
 ; ZVFHMIN-NEXT:    slli a1, a1, 3
 ; ZVFHMIN-NEXT:    add a1, a1, a3
 ; ZVFHMIN-NEXT:    add a1, sp, a1
@@ -3611,6 +3619,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    slli t0, t0, 1
 ; ZVFHMIN-NEXT:    mv t1, t0
 ; ZVFHMIN-NEXT:    slli t0, t0, 2
+; ZVFHMIN-NEXT:    add t1, t1, t0
+; ZVFHMIN-NEXT:    slli t0, t0, 1
 ; ZVFHMIN-NEXT:    add t0, t0, t1
 ; ZVFHMIN-NEXT:    add t0, sp, t0
 ; ZVFHMIN-NEXT:    addi t0, t0, 16
@@ -3618,9 +3628,8 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vslidedown.vx v16, v8, a1
 ; ZVFHMIN-NEXT:    vl8re16.v v8, (a0)
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    mv t0, a0
 ; ZVFHMIN-NEXT:    slli a0, a0, 1
-; ZVFHMIN-NEXT:    add t0, t0, a0
+; ZVFHMIN-NEXT:    mv t0, a0
 ; ZVFHMIN-NEXT:    slli a0, a0, 2
 ; ZVFHMIN-NEXT:    add a0, a0, t0
 ; ZVFHMIN-NEXT:    add a0, sp, a0
@@ -3650,10 +3659,6 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v5, v8, v16, v0.t
-; ZVFHMIN-NEXT:    csrr a0, vlenb
-; ZVFHMIN-NEXT:    add a0, sp, a0
-; ZVFHMIN-NEXT:    addi a0, a0, 16
-; ZVFHMIN-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; ZVFHMIN-NEXT:    vsetvli zero, a6, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v24
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
@@ -3662,85 +3667,95 @@ define <vscale x 64 x i1> @fcmp_oeq_vv_nxv64f16(<vscale x 64 x half> %va, <vscal
 ; ZVFHMIN-NEXT:    addi a0, a0, 16
 ; ZVFHMIN-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
+; ZVFHMIN-NEXT:    csrr a0, vlenb
+; ZVFHMIN-NEXT:    add a0, sp, a0
+; ZVFHMIN-NEXT:    addi a0, a0, 16
+; ZVFHMIN-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v6, v24, v16, v0.t
-; ZVFHMIN-NEXT:    add a0, a3, a3
+; ZVFHMIN-NEXT:    vmfeq.vv v7, v24, v16, v0.t
 ; ZVFHMIN-NEXT:    bltu a2, a5, .LBB171_4
 ; ZVFHMIN-NEXT:  # %bb.3:
 ; ZVFHMIN-NEXT:    mv a2, a5
 ; ZVFHMIN-NEXT:  .LBB171_4:
-; ZVFHMIN-NEXT:    sub a5, a2, a4
-; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    slli a6, a6, 1
-; ZVFHMIN-NEXT:    mv a7, a6
-; ZVFHMIN-NEXT:    slli a6, a6, 2
-; ZVFHMIN-NEXT:    add a6, a6, a7
-; ZVFHMIN-NEXT:    add a6, sp, a6
-; ZVFHMIN-NEXT:    addi a6, a6, 16
-; ZVFHMIN-NEXT:    vl1r.v v7, (a6) # vscale x 8-byte Folded Reload
-; ZVFHMIN-NEXT:    vsetvli a6, zero, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslidedown.vx v0, v7, a3
-; ZVFHMIN-NEXT:    sltu a6, a2, a5
-; ZVFHMIN-NEXT:    addi a6, a6, -1
-; ZVFHMIN-NEXT:    and a5, a6, a5
-; ZVFHMIN-NEXT:    csrr a6, vlenb
-; ZVFHMIN-NEXT:    mv a7, a6
-; ZVFHMIN-NEXT:    slli a6, a6, 1
-; ZVFHMIN-NEXT:    add a7, a7, a6
-; ZVFHMIN-NEXT:    slli a6, a6, 3
-; ZVFHMIN-NEXT:    add a6, a6, a7
-; ZVFHMIN-NEXT:    add a6, sp, a6
-; ZVFHMIN-NEXT:    addi a6, a6, 16
-; ZVFHMIN-NEXT:    vl8r.v v24, (a6) # vscale x 64-byte Folded Reload
-; ZVFHMIN-NEXT:    vsetvli zero, a5, e16, m4, ta, ma
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v28
+; ZVFHMIN-NEXT:    sub a0, a2, a4
 ; ZVFHMIN-NEXT:    csrr a5, vlenb
-; ZVFHMIN-NEXT:    mv a6, a5
 ; ZVFHMIN-NEXT:    slli a5, a5, 1
-; ZVFHMIN-NEXT:    add a6, a6, a5
+; ZVFHMIN-NEXT:    mv a6, a5
 ; ZVFHMIN-NEXT:    slli a5, a5, 2
+; ZVFHMIN-NEXT:    add a6, a6, a5
+; ZVFHMIN-NEXT:    slli a5, a5, 1
+; ZVFHMIN-NEXT:    add a5, a5, a6
+; ZVFHMIN-NEXT:    add a5, sp, a5
+; ZVFHMIN-NEXT:    addi a5, a5, 16
+; ZVFHMIN-NEXT:    vl1r.v v8, (a5) # vscale x 8-byte Folded Reload
+; ZVFHMIN-NEXT:    vsetvli a5, zero, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslidedown.vx v0, v8, a3
+; ZVFHMIN-NEXT:    sltu a5, a2, a0
+; ZVFHMIN-NEXT:    addi a5, a5, -1
+; ZVFHMIN-NEXT:    and a0, a5, a0
+; ZVFHMIN-NEXT:    csrr a5, vlenb
+; ZVFHMIN-NEXT:    slli a5, a5, 1
+; ZVFHMIN-NEXT:    mv a6, a5
+; ZVFHMIN-NEXT:    slli a5, a5, 3
 ; ZVFHMIN-NEXT:    add a5, a5, a6
 ; ZVFHMIN-NEXT:    add a5, sp, a5
 ; ZVFHMIN-NEXT:    addi a5, a5, 16
-; ZVFHMIN-NEXT:    vl8r.v v8, (a5) # vscale x 64-byte Folded Reload
+; ZVFHMIN-NEXT:    vl8r.v v24, (a5) # vscale x 64-byte Folded Reload
+; ZVFHMIN-NEXT:    vsetvli zero, a0, e16, m4, ta, ma
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v28
+; ZVFHMIN-NEXT:    csrr a0, vlenb
+; ZVFHMIN-NEXT:    slli a0, a0, 1
+; ZVFHMIN-NEXT:    mv a5, a0
+; ZVFHMIN-NEXT:    slli a0, a0, 2
+; ZVFHMIN-NEXT:    add a0, a0, a5
+; ZVFHMIN-NEXT:    add a0, sp, a0
+; ZVFHMIN-NEXT:    addi a0, a0, 16
+; ZVFHMIN-NEXT:    vl8r.v v8, (a0) # vscale x 64-byte Folded Reload
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v12
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
-; ZVFHMIN-NEXT:    vmfeq.vv v4, v16, v24, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslideup.vx v6, v5, a3
+; ZVFHMIN-NEXT:    vmfeq.vv v10, v16, v24, v0.t
+; ZVFHMIN-NEXT:    vmv1r.v v9, v7
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslideup.vx v9, v5, a3
 ; ZVFHMIN-NEXT:    bltu a2, a4, .LBB171_6
 ; ZVFHMIN-NEXT:  # %bb.5:
 ; ZVFHMIN-NEXT:    mv a2, a4
 ; ZVFHMIN-NEXT:  .LBB171_6:
-; ZVFHMIN-NEXT:    vmv1r.v v0, v7
-; ZVFHMIN-NEXT:    csrr a4, vlenb
-; ZVFHMIN-NEXT:    mv a5, a4
-; ZVFHMIN-NEXT:    slli a4, a4, 1
-; ZVFHMIN-NEXT:    add a5, a5, a4
-; ZVFHMIN-NEXT:    slli a4, a4, 3
-; ZVFHMIN-NEXT:    add a4, a4, a5
-; ZVFHMIN-NEXT:    add a4, sp, a4
-; ZVFHMIN-NEXT:    addi a4, a4, 16
-; ZVFHMIN-NEXT:    vl8r.v v24, (a4) # vscale x 64-byte Folded Reload
+; ZVFHMIN-NEXT:    csrr a0, vlenb
+; ZVFHMIN-NEXT:    slli a0, a0, 1
+; ZVFHMIN-NEXT:    mv a4, a0
+; ZVFHMIN-NEXT:    slli a0, a0, 3
+; ZVFHMIN-NEXT:    add a0, a0, a4
+; ZVFHMIN-NEXT:    add a0, sp, a0
+; ZVFHMIN-NEXT:    addi a0, a0, 16
+; ZVFHMIN-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; ZVFHMIN-NEXT:    vsetvli zero, a2, e16, m4, ta, ma
 ; ZVFHMIN-NEXT:    vfwcvt.f.f.v v16, v24
-; ZVFHMIN-NEXT:    csrr a2, vlenb
-; ZVFHMIN-NEXT:    mv a4, a2
-; ZVFHMIN-NEXT:    slli a2, a2, 1
-; ZVFHMIN-NEXT:    add a4, a4, a2
-; ZVFHMIN-NEXT:    slli a2, a2, 2
-; ZVFHMIN-NEXT:    add a2, a2, a4
-; ZVFHMIN-NEXT:    add a2, sp, a2
-; ZVFHMIN-NEXT:    addi a2, a2, 16
-; ZVFHMIN-NEXT:    vl8r.v v8, (a2) # vscale x 64-byte Folded Reload
-; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v8
+; ZVFHMIN-NEXT:    csrr a0, vlenb
+; ZVFHMIN-NEXT:    slli a0, a0, 1
+; ZVFHMIN-NEXT:    mv a2, a0
+; ZVFHMIN-NEXT:    slli a0, a0, 2
+; ZVFHMIN-NEXT:    add a0, a0, a2
+; ZVFHMIN-NEXT:    add a0, sp, a0
+; ZVFHMIN-NEXT:    addi a0, a0, 16
+; ZVFHMIN-NEXT:    vl8r.v v0, (a0) # vscale x 64-byte Folded Reload
+; ZVFHMIN-NEXT:    vfwcvt.f.f.v v24, v0
+; ZVFHMIN-NEXT:    csrr a0, vlenb
+; ZVFHMIN-NEXT:    slli a0, a0, 1
+; ZVFHMIN-NEXT:    mv a2, a0
+; ZVFHMIN-NEXT:    slli a0, a0, 2
+; ZVFHMIN-NEXT:    add a2, a2, a0
+; ZVFHMIN-NEXT:    slli a0, a0, 1
+; ZVFHMIN-NEXT:    add a0, a0, a2
+; ZVFHMIN-NEXT:    add a0, sp, a0
+; ZVFHMIN-NEXT:    addi a0, a0, 16
+; ZVFHMIN-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
 ; ZVFHMIN-NEXT:    vsetvli zero, zero, e32, m8, ta, ma
 ; ZVFHMIN-NEXT:    vmfeq.vv v8, v16, v24, v0.t
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; ZVFHMIN-NEXT:    vslideup.vx v8, v4, a3
-; ZVFHMIN-NEXT:    add a0, a1, a1
-; ZVFHMIN-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; ZVFHMIN-NEXT:    vslideup.vx v8, v6, a1
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v10, a3
+; ZVFHMIN-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVFHMIN-NEXT:    vslideup.vx v8, v9, a1
 ; ZVFHMIN-NEXT:    vmv.v.v v0, v8
 ; ZVFHMIN-NEXT:    csrr a0, vlenb
 ; ZVFHMIN-NEXT:    mv a1, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
index ae868fed68ca..ff923efe8eb4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-fp.ll
@@ -4280,8 +4280,7 @@ define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
 ; RV32-NEXT:    vmfeq.vf v24, v16, fa5
 ; RV32-NEXT:    vmfeq.vf v0, v8, fa5
 ; RV32-NEXT:    srli a0, a0, 3
-; RV32-NEXT:    add a1, a0, a0
-; RV32-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; RV32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vslideup.vx v0, v24, a0
 ; RV32-NEXT:    ret
 ;
@@ -4293,8 +4292,7 @@ define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
 ; RV64-NEXT:    vmfeq.vf v24, v16, fa5
 ; RV64-NEXT:    vmfeq.vf v0, v8, fa5
 ; RV64-NEXT:    srli a0, a0, 3
-; RV64-NEXT:    add a1, a0, a0
-; RV64-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vslideup.vx v0, v24, a0
 ; RV64-NEXT:    ret
 ;
@@ -4306,8 +4304,7 @@ define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
 ; ZVFHMIN32-NEXT:    vmfeq.vf v24, v16, fa5
 ; ZVFHMIN32-NEXT:    vmfeq.vf v0, v8, fa5
 ; ZVFHMIN32-NEXT:    srli a0, a0, 3
-; ZVFHMIN32-NEXT:    add a1, a0, a0
-; ZVFHMIN32-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; ZVFHMIN32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; ZVFHMIN32-NEXT:    vslideup.vx v0, v24, a0
 ; ZVFHMIN32-NEXT:    ret
 ;
@@ -4319,8 +4316,7 @@ define <vscale x 16 x i1> @fcmp_oeq_vf_nx16f64(<vscale x 16 x double> %va) {
 ; ZVFHMIN64-NEXT:    vmfeq.vf v24, v16, fa5
 ; ZVFHMIN64-NEXT:    vmfeq.vf v0, v8, fa5
 ; ZVFHMIN64-NEXT:    srli a0, a0, 3
-; ZVFHMIN64-NEXT:    add a1, a0, a0
-; ZVFHMIN64-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; ZVFHMIN64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; ZVFHMIN64-NEXT:    vslideup.vx v0, v24, a0
 ; ZVFHMIN64-NEXT:    ret
   %vc = fcmp oeq <vscale x 16 x double> %va, zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
index ef560a7631de..13c63d9c80a9 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-int-vp.ll
@@ -2246,8 +2246,7 @@ define <vscale x 32 x i1> @icmp_eq_vv_nxv32i32(<vscale x 32 x i32> %va, <vscale
 ; CHECK-NEXT:    vl8r.v v24, (a0) # vscale x 64-byte Folded Reload
 ; CHECK-NEXT:    vsetvli zero, a2, e32, m8, ta, ma
 ; CHECK-NEXT:    vmseq.vv v16, v24, v8, v0.t
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v16, v6, a1
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    csrr a0, vlenb
@@ -2283,8 +2282,7 @@ define <vscale x 32 x i1> @icmp_eq_vx_nxv32i32(<vscale x 32 x i32> %va, i32 %b,
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vmseq.vx v16, v8, a0, v0.t
-; CHECK-NEXT:    add a0, a2, a2
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v16, v25, a2
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    ret
@@ -2316,8 +2314,7 @@ define <vscale x 32 x i1> @icmp_eq_vx_swap_nxv32i32(<vscale x 32 x i32> %va, i32
 ; CHECK-NEXT:    vmv1r.v v0, v24
 ; CHECK-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
 ; CHECK-NEXT:    vmseq.vx v16, v8, a0, v0.t
-; CHECK-NEXT:    add a0, a2, a2
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v16, v25, a2
 ; CHECK-NEXT:    vmv1r.v v0, v16
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
index bd3c29b0c6ef..a85b471530cc 100644
--- a/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/setcc-integer.ll
@@ -3001,9 +3001,8 @@ define <vscale x 16 x i1> @icmp_eq_vi_nx16i64(<vscale x 16 x i64> %va) {
 ; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
 ; CHECK-NEXT:    vmseq.vi v24, v16, 0
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vmseq.vi v0, v8, 0
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf4, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
 ; CHECK-NEXT:    vslideup.vx v0, v24, a0
 ; CHECK-NEXT:    ret
   %vc = icmp eq <vscale x 16 x i64> %va, zeroinitializer
diff --git a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
index c9f9a7973300..790cd56ee952 100644
--- a/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/undef-earlyclobber-chain.ll
@@ -48,10 +48,10 @@ define internal void @SubRegLivenessUndefInPhi(i64 %cond) {
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf4, ta, ma
 ; CHECK-NEXT:    vid.v v9
 ; CHECK-NEXT:    csrr a0, vlenb
+; CHECK-NEXT:    srli a1, a0, 2
 ; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    vadd.vi v10, v9, 1
 ; CHECK-NEXT:    vadd.vi v11, v9, 3
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    vslideup.vx v12, v10, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
index baace6d26f14..4753ab915bdf 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll
@@ -191,8 +191,7 @@ define {<2 x i32>, <2 x i32>, <2 x i32>} @vector_deinterleave3_v2i32_v6i32(<6 x
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 4
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -222,8 +221,7 @@ define {<2 x i32>, <2 x i32>, <2 x i32>, <2 x i32>} @vector_deinterleave4_v2i32_
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v9, v8, 2
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v12, v10, a0
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    addi a0, sp, 16
@@ -254,15 +252,13 @@ define {<2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>} @vector_deinterle
 ; CHECK-NEXT:    vslidedown.vi v14, v8, 2
 ; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 8
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v13, v12, a1
-; CHECK-NEXT:    vslideup.vx v8, v14, a1
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v13, a0
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v13, v12, a0
+; CHECK-NEXT:    vslideup.vx v8, v14, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v13, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vs2r.v v8, (a0)
@@ -292,16 +288,14 @@ define {<2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>, <2 x i16>} @vecto
 ; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 10
 ; CHECK-NEXT:    vslidedown.vi v12, v8, 8
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    add a3, a0, a0
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v15, v14, a1
-; CHECK-NEXT:    vslideup.vx v8, v16, a1
-; CHECK-NEXT:    vslideup.vx v12, v10, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v15, a0
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v15, v14, a0
+; CHECK-NEXT:    vslideup.vx v8, v16, a0
+; CHECK-NEXT:    vslideup.vx v12, v10, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v15, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v12
 ; CHECK-NEXT:    vs2r.v v8, (a0)
@@ -330,22 +324,19 @@ define {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>} @v
 ; CHECK-NEXT:    vslidedown.vi v12, v8, 2
 ; CHECK-NEXT:    vslidedown.vi v13, v8, 4
 ; CHECK-NEXT:    vslidedown.vi v14, v8, 6
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a2, a0, 2
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a2, a0, 3
 ; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    add a3, a1, a1
-; CHECK-NEXT:    add a4, a2, a1
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vx v10, v9, a2
+; CHECK-NEXT:    add a3, a1, a2
+; CHECK-NEXT:    vslideup.vx v8, v12, a2
 ; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vx v10, v9, a1
-; CHECK-NEXT:    vslideup.vx v8, v12, a1
-; CHECK-NEXT:    add a3, a0, a0
-; CHECK-NEXT:    add a1, a4, a1
-; CHECK-NEXT:    vsetvli zero, a4, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vx v10, v11, a2
-; CHECK-NEXT:    vslideup.vx v8, v13, a2
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v14, a4
-; CHECK-NEXT:    vsetvli zero, a3, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v11, a1
+; CHECK-NEXT:    vslideup.vx v8, v13, a1
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v14, a3
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs1r.v v8, (a0)
@@ -374,23 +365,20 @@ define {<2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2 x i8>, <2
 ; CHECK-NEXT:    vslidedown.vi v13, v8, 2
 ; CHECK-NEXT:    vslidedown.vi v14, v8, 4
 ; CHECK-NEXT:    vslidedown.vi v15, v8, 6
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a2, a0, 2
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a2, a0, 3
 ; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    add a3, a1, a1
-; CHECK-NEXT:    add a4, a2, a1
-; CHECK-NEXT:    add a5, a0, a0
+; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, tu, ma
+; CHECK-NEXT:    vslideup.vx v10, v9, a2
+; CHECK-NEXT:    add a3, a1, a2
+; CHECK-NEXT:    vslideup.vx v8, v13, a2
 ; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vx v10, v9, a1
-; CHECK-NEXT:    vslideup.vx v8, v13, a1
-; CHECK-NEXT:    add a1, a4, a1
-; CHECK-NEXT:    vsetvli zero, a4, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vx v10, v11, a2
-; CHECK-NEXT:    vslideup.vx v8, v14, a2
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v10, v12, a4
-; CHECK-NEXT:    vslideup.vx v8, v15, a4
-; CHECK-NEXT:    vsetvli zero, a5, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v11, a1
+; CHECK-NEXT:    vslideup.vx v8, v14, a1
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v12, a3
+; CHECK-NEXT:    vslideup.vx v8, v15, a3
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs1r.v v8, (a0)
@@ -551,8 +539,7 @@ define {<2 x float>, <2 x float>, <2 x float>} @vector_deinterleave3_v6f32_v2f32
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 4
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v10
@@ -586,8 +573,7 @@ define {<2 x float>, <2 x float>, <2 x float>, <2 x float>} @vector_deinterleave
 ; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v9, v8, 2
 ; CHECK-NEXT:    srli a0, a0, 3
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v12, v10, a0
 ; CHECK-NEXT:    vslideup.vx v8, v9, a0
 ; CHECK-NEXT:    addi a0, sp, 16
@@ -622,15 +608,13 @@ define {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>} @vector_dein
 ; CHECK-NEXT:    vslidedown.vi v14, v8, 2
 ; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 8
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v13, v12, a1
-; CHECK-NEXT:    vslideup.vx v8, v14, a1
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v13, a0
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v13, v12, a0
+; CHECK-NEXT:    vslideup.vx v8, v14, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v13, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v10
 ; CHECK-NEXT:    vs2r.v v8, (a0)
@@ -664,16 +648,14 @@ define {<2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>, <2 x half>}
 ; CHECK-NEXT:    vsetivli zero, 2, e16, m2, ta, ma
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 10
 ; CHECK-NEXT:    vslidedown.vi v12, v8, 8
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    add a3, a0, a0
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v15, v14, a1
-; CHECK-NEXT:    vslideup.vx v8, v16, a1
-; CHECK-NEXT:    vslideup.vx v12, v10, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v15, a0
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v15, v14, a0
+; CHECK-NEXT:    vslideup.vx v8, v16, a0
+; CHECK-NEXT:    vslideup.vx v12, v10, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v15, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vmv1r.v v9, v12
 ; CHECK-NEXT:    vs2r.v v8, (a0)
@@ -707,21 +689,18 @@ define {<1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>,
 ; CHECK-NEXT:    vmv1r.v v10, v8
 ; CHECK-NEXT:    vslidedown.vi v13, v8, 5
 ; CHECK-NEXT:    vslidedown.vi v14, v8, 6
-; CHECK-NEXT:    srli a1, a0, 3
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    add a3, a0, a0
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v11, v9, a1
-; CHECK-NEXT:    vslideup.vx v10, v12, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v10, v11, a0
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v9, a0
+; CHECK-NEXT:    vslideup.vx v10, v12, a0
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v11, a1
 ; CHECK-NEXT:    vslidedown.vi v11, v8, 4
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v11, v13, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v11, v14, a0
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v13, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v14, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs2r.v v10, (a0)
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
@@ -751,25 +730,22 @@ define {<1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>, <1 x half>,
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 7
 ; CHECK-NEXT:    vslidedown.vi v11, v8, 6
 ; CHECK-NEXT:    vslidedown.vi v12, v8, 5
-; CHECK-NEXT:    srli a1, a0, 3
+; CHECK-NEXT:    srli a1, a0, 2
+; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    vslidedown.vi v9, v8, 4
-; CHECK-NEXT:    srli a0, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    add a3, a0, a0
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v11, v10, a1
-; CHECK-NEXT:    vslideup.vx v9, v12, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v9, v11, a0
-; CHECK-NEXT:    vsetivli zero, 1, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v10, a0
+; CHECK-NEXT:    vslideup.vx v9, v12, a0
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v9, v11, a1
 ; CHECK-NEXT:    vslidedown.vi v10, v8, 3
 ; CHECK-NEXT:    vslidedown.vi v11, v8, 2
 ; CHECK-NEXT:    vslidedown.vi v12, v8, 1
-; CHECK-NEXT:    vsetvli zero, a2, e16, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v11, v10, a1
-; CHECK-NEXT:    vslideup.vx v8, v12, a1
-; CHECK-NEXT:    vsetvli zero, a3, e16, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v11, a0
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v11, v10, a0
+; CHECK-NEXT:    vslideup.vx v8, v12, a0
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v11, a1
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs2r.v v8, (a0)
 ; CHECK-NEXT:    vsetvli a1, zero, e16, mf4, ta, ma
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
index 75f92c86ff09..6144f916ea52 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave.ll
@@ -2705,16 +2705,10 @@ define {<vscale x 2 x half>, <vscale x 2 x half>, <vscale x 2 x half>, <vscale x
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v9, a0
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v8, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v10, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v11, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs4r.v v8, (a0)
@@ -2801,16 +2795,10 @@ define {<vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vscale x 2 x bfloat>, <vs
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v9, a0
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v8, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v10, a0
-; CHECK-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v11, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs4r.v v8, (a0)
@@ -2897,16 +2885,10 @@ define {<vscale x 1 x float>, <vscale x 1 x float>, <vscale x 1 x float>, <vscal
 ; CHECK-NEXT:    srli a0, a0, 3
 ; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v9, a0
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v8, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a0
-; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslidedown.vx v11, v10, a0
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v11, a0
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    vs4r.v v8, (a0)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
index 3751967f18aa..a5811e697634 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave-store.ll
@@ -14,18 +14,17 @@ define void @vector_interleave_store_nxv32i1_nxv16i1(<vscale x 16 x i1> %a, <vsc
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmv.v.i v10, 0
 ; CHECK-NEXT:    li a1, -1
-; CHECK-NEXT:    csrr a2, vlenb
 ; CHECK-NEXT:    vmerge.vim v12, v10, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v14, v10, 1, v0
-; CHECK-NEXT:    srli a2, a2, 2
 ; CHECK-NEXT:    vwaddu.vv v8, v14, v12
 ; CHECK-NEXT:    vwmaccu.vx v8, a1, v12
+; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    vmsne.vi v12, v10, 0
 ; CHECK-NEXT:    vmsne.vi v10, v8, 0
-; CHECK-NEXT:    add a1, a2, a2
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v10, v12, a2
+; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v10, v12, a1
 ; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
 ; CHECK-NEXT:    vsm.v v10, (a0)
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
index e297e88c71f1..01cc5c58b24c 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vector-interleave.ll
@@ -17,18 +17,17 @@ define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1>
 ; V-NEXT:    vmv1r.v v0, v8
 ; V-NEXT:    vmv.v.i v10, 0
 ; V-NEXT:    li a0, -1
-; V-NEXT:    csrr a1, vlenb
 ; V-NEXT:    vmerge.vim v12, v10, 1, v0
 ; V-NEXT:    vmv1r.v v0, v9
 ; V-NEXT:    vmerge.vim v14, v10, 1, v0
-; V-NEXT:    srli a1, a1, 2
 ; V-NEXT:    vwaddu.vv v8, v14, v12
 ; V-NEXT:    vwmaccu.vx v8, a0, v12
+; V-NEXT:    csrr a0, vlenb
 ; V-NEXT:    vmsne.vi v12, v10, 0
 ; V-NEXT:    vmsne.vi v0, v8, 0
-; V-NEXT:    add a0, a1, a1
-; V-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; V-NEXT:    vslideup.vx v0, v12, a1
+; V-NEXT:    srli a0, a0, 2
+; V-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; V-NEXT:    vslideup.vx v0, v12, a0
 ; V-NEXT:    ret
 ;
 ; ZVBB-LABEL: vector_interleave_nxv32i1_nxv16i1:
@@ -38,17 +37,16 @@ define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    vmv1r.v v0, v8
 ; ZVBB-NEXT:    vmv.v.i v10, 0
 ; ZVBB-NEXT:    li a0, 1
-; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    vmerge.vim v10, v10, 1, v0
-; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    vwsll.vi v12, v10, 8
 ; ZVBB-NEXT:    vmv1r.v v0, v9
 ; ZVBB-NEXT:    vwaddu.wx v12, v12, a0, v0.t
+; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    vmsne.vi v8, v14, 0
 ; ZVBB-NEXT:    vmsne.vi v0, v12, 0
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v8, a1
+; ZVBB-NEXT:    srli a0, a0, 2
+; ZVBB-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v8, a0
 ; ZVBB-NEXT:    ret
 ;
 ; ZIP-LABEL: vector_interleave_nxv32i1_nxv16i1:
@@ -61,13 +59,12 @@ define <vscale x 32 x i1> @vector_interleave_nxv32i1_nxv16i1(<vscale x 16 x i1>
 ; ZIP-NEXT:    vmerge.vim v12, v10, 1, v0
 ; ZIP-NEXT:    vmv1r.v v0, v9
 ; ZIP-NEXT:    vmerge.vim v8, v10, 1, v0
-; ZIP-NEXT:    srli a0, a0, 2
 ; ZIP-NEXT:    ri.vzip2b.vv v10, v8, v12
 ; ZIP-NEXT:    ri.vzip2a.vv v14, v8, v12
 ; ZIP-NEXT:    vmsne.vi v8, v10, 0
 ; ZIP-NEXT:    vmsne.vi v0, v14, 0
-; ZIP-NEXT:    add a1, a0, a0
-; ZIP-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; ZIP-NEXT:    srli a0, a0, 2
+; ZIP-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZIP-NEXT:    vslideup.vx v0, v8, a0
 ; ZIP-NEXT:    ret
   %res = call <vscale x 32 x i1> @llvm.vector.interleave2.nxv32i1(<vscale x 16 x i1> %a, <vscale x 16 x i1> %b)
@@ -508,19 +505,17 @@ define <vscale x 48 x i1> @vector_interleave_nxv48i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    add a2, a3, a2
 ; CHECK-NEXT:    vsseg3e8.v v14, (a0)
 ; CHECK-NEXT:    vl2r.v v8, (a2)
-; CHECK-NEXT:    srli a2, a1, 2
-; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    srli a2, a1, 1
 ; CHECK-NEXT:    vl2r.v v10, (a3)
 ; CHECK-NEXT:    vl2r.v v12, (a0)
-; CHECK-NEXT:    add a0, a2, a2
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    vmsne.vi v14, v8, 0
 ; CHECK-NEXT:    vmsne.vi v8, v10, 0
 ; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v8, a2
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v14, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v14, a2
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 6
 ; CHECK-NEXT:    mul a0, a0, a1
@@ -551,19 +546,17 @@ define <vscale x 48 x i1> @vector_interleave_nxv48i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    add a2, a3, a2
 ; ZVBB-NEXT:    vsseg3e8.v v14, (a0)
 ; ZVBB-NEXT:    vl2r.v v8, (a2)
-; ZVBB-NEXT:    srli a2, a1, 2
-; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    srli a2, a1, 1
 ; ZVBB-NEXT:    vl2r.v v10, (a3)
 ; ZVBB-NEXT:    vl2r.v v12, (a0)
-; ZVBB-NEXT:    add a0, a2, a2
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    vmsne.vi v14, v8, 0
 ; ZVBB-NEXT:    vmsne.vi v8, v10, 0
 ; ZVBB-NEXT:    vmsne.vi v0, v12, 0
-; ZVBB-NEXT:    vsetvli zero, a0, e8, mf2, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v8, a2
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v14, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v8, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v14, a2
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    li a1, 6
 ; ZVBB-NEXT:    mul a0, a0, a1
@@ -812,22 +805,20 @@ define <vscale x 64 x i1> @vector_interleave_nxv64i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    add a2, a4, a2
 ; CHECK-NEXT:    vsseg4e8.v v14, (a0)
 ; CHECK-NEXT:    vl2r.v v8, (a2)
-; CHECK-NEXT:    srli a2, a1, 2
-; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    srli a2, a1, 1
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    vl2r.v v10, (a4)
-; CHECK-NEXT:    add a4, a2, a2
 ; CHECK-NEXT:    vl2r.v v12, (a3)
 ; CHECK-NEXT:    vl2r.v v14, (a0)
 ; CHECK-NEXT:    vmsne.vi v16, v8, 0
 ; CHECK-NEXT:    vmsne.vi v8, v10, 0
 ; CHECK-NEXT:    vmsne.vi v9, v12, 0
 ; CHECK-NEXT:    vmsne.vi v0, v14, 0
-; CHECK-NEXT:    vsetvli zero, a4, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v8, v16, a2
-; CHECK-NEXT:    vslideup.vx v0, v9, a2
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v8, v16, a1
+; CHECK-NEXT:    vslideup.vx v0, v9, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v8, a2
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 3
 ; CHECK-NEXT:    add sp, sp, a0
@@ -859,22 +850,20 @@ define <vscale x 64 x i1> @vector_interleave_nxv64i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    add a2, a4, a2
 ; ZVBB-NEXT:    vsseg4e8.v v14, (a0)
 ; ZVBB-NEXT:    vl2r.v v8, (a2)
-; ZVBB-NEXT:    srli a2, a1, 2
-; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    srli a2, a1, 1
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    vl2r.v v10, (a4)
-; ZVBB-NEXT:    add a4, a2, a2
 ; ZVBB-NEXT:    vl2r.v v12, (a3)
 ; ZVBB-NEXT:    vl2r.v v14, (a0)
 ; ZVBB-NEXT:    vmsne.vi v16, v8, 0
 ; ZVBB-NEXT:    vmsne.vi v8, v10, 0
 ; ZVBB-NEXT:    vmsne.vi v9, v12, 0
 ; ZVBB-NEXT:    vmsne.vi v0, v14, 0
-; ZVBB-NEXT:    vsetvli zero, a4, e8, mf2, ta, ma
-; ZVBB-NEXT:    vslideup.vx v8, v16, a2
-; ZVBB-NEXT:    vslideup.vx v0, v9, a2
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e8, m1, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v8, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v8, v16, a1
+; ZVBB-NEXT:    vslideup.vx v0, v9, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v8, a2
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 3
 ; ZVBB-NEXT:    add sp, sp, a0
@@ -1114,7 +1103,7 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v18, v12, 1, v0
 ; CHECK-NEXT:    add a2, a4, a1
-; CHECK-NEXT:    srli a3, a1, 2
+; CHECK-NEXT:    srli a3, a1, 1
 ; CHECK-NEXT:    vmv2r.v v20, v14
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v16, v12, 1, v0
@@ -1144,11 +1133,9 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    add a5, a4, a1
 ; CHECK-NEXT:    vl1r.v v16, (a5)
 ; CHECK-NEXT:    add a5, a5, a1
-; CHECK-NEXT:    srli a1, a1, 1
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    vl1r.v v11, (a2)
-; CHECK-NEXT:    add a2, a3, a3
 ; CHECK-NEXT:    vl1r.v v15, (a4)
-; CHECK-NEXT:    add a4, a1, a1
 ; CHECK-NEXT:    vl1r.v v13, (a0)
 ; CHECK-NEXT:    vl1r.v v17, (a5)
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -1156,11 +1143,11 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0
 ; CHECK-NEXT:    vmsne.vi v8, v14, 0
 ; CHECK-NEXT:    vmsne.vi v9, v12, 0
-; CHECK-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v18, a3
-; CHECK-NEXT:    vslideup.vx v9, v8, a3
-; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v9, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v18, a1
+; CHECK-NEXT:    vslideup.vx v9, v8, a1
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v9, a3
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmsne.vi v8, v16, 0
 ; CHECK-NEXT:    csrr a0, vlenb
@@ -1190,7 +1177,7 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    vmv1r.v v0, v8
 ; ZVBB-NEXT:    vmerge.vim v18, v12, 1, v0
 ; ZVBB-NEXT:    add a2, a4, a1
-; ZVBB-NEXT:    srli a3, a1, 2
+; ZVBB-NEXT:    srli a3, a1, 1
 ; ZVBB-NEXT:    vmv2r.v v20, v14
 ; ZVBB-NEXT:    vmv1r.v v0, v9
 ; ZVBB-NEXT:    vmerge.vim v16, v12, 1, v0
@@ -1220,11 +1207,9 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    add a5, a4, a1
 ; ZVBB-NEXT:    vl1r.v v16, (a5)
 ; ZVBB-NEXT:    add a5, a5, a1
-; ZVBB-NEXT:    srli a1, a1, 1
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    vl1r.v v11, (a2)
-; ZVBB-NEXT:    add a2, a3, a3
 ; ZVBB-NEXT:    vl1r.v v15, (a4)
-; ZVBB-NEXT:    add a4, a1, a1
 ; ZVBB-NEXT:    vl1r.v v13, (a0)
 ; ZVBB-NEXT:    vl1r.v v17, (a5)
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -1232,11 +1217,11 @@ define <vscale x 80 x i1> @vector_interleave_nxv80i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    vmsne.vi v0, v10, 0
 ; ZVBB-NEXT:    vmsne.vi v8, v14, 0
 ; ZVBB-NEXT:    vmsne.vi v9, v12, 0
-; ZVBB-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v18, a3
-; ZVBB-NEXT:    vslideup.vx v9, v8, a3
-; ZVBB-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v9, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v18, a1
+; ZVBB-NEXT:    vslideup.vx v9, v8, a1
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v9, a3
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; ZVBB-NEXT:    vmsne.vi v8, v16, 0
 ; ZVBB-NEXT:    csrr a0, vlenb
@@ -2340,47 +2325,45 @@ define <vscale x 96 x i1> @vector_interleave_nxv96i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    vmv1r.v v17, v9
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v24, v20, 1, v0
-; CHECK-NEXT:    addi a5, sp, 16
+; CHECK-NEXT:    addi a4, sp, 16
 ; CHECK-NEXT:    vmv1r.v v18, v25
 ; CHECK-NEXT:    vmv1r.v v0, v11
 ; CHECK-NEXT:    vmerge.vim v26, v20, 1, v0
-; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    vmv1r.v v19, v27
 ; CHECK-NEXT:    vmv1r.v v0, v12
 ; CHECK-NEXT:    vmerge.vim v10, v20, 1, v0
-; CHECK-NEXT:    add a3, a0, a2
+; CHECK-NEXT:    add a2, a0, a1
 ; CHECK-NEXT:    vmv1r.v v20, v11
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vsseg6e8.v v15, (a0)
 ; CHECK-NEXT:    vmv1r.v v15, v22
-; CHECK-NEXT:    add a4, a5, a2
+; CHECK-NEXT:    add a5, a4, a1
 ; CHECK-NEXT:    vmv1r.v v16, v8
-; CHECK-NEXT:    srli a1, a2, 2
+; CHECK-NEXT:    srli a3, a1, 1
 ; CHECK-NEXT:    vmv1r.v v17, v24
-; CHECK-NEXT:    add a6, a4, a2
+; CHECK-NEXT:    add a6, a5, a1
 ; CHECK-NEXT:    vmv1r.v v18, v26
-; CHECK-NEXT:    add a7, a3, a2
+; CHECK-NEXT:    add a7, a2, a1
 ; CHECK-NEXT:    vmv1r.v v19, v10
-; CHECK-NEXT:    vsseg6e8.v v14, (a5)
+; CHECK-NEXT:    vsseg6e8.v v14, (a4)
 ; CHECK-NEXT:    vl1r.v v8, (a0)
-; CHECK-NEXT:    add a0, a6, a2
+; CHECK-NEXT:    add a0, a6, a1
 ; CHECK-NEXT:    vl1r.v v10, (a6)
-; CHECK-NEXT:    add a6, a7, a2
-; CHECK-NEXT:    vl1r.v v12, (a5)
-; CHECK-NEXT:    add a5, a0, a2
+; CHECK-NEXT:    add a6, a7, a1
+; CHECK-NEXT:    vl1r.v v12, (a4)
+; CHECK-NEXT:    add a4, a0, a1
 ; CHECK-NEXT:    vl1r.v v14, (a7)
-; CHECK-NEXT:    add a7, a6, a2
-; CHECK-NEXT:    vl1r.v v16, (a5)
-; CHECK-NEXT:    add a5, a5, a2
+; CHECK-NEXT:    add a7, a6, a1
+; CHECK-NEXT:    vl1r.v v16, (a4)
+; CHECK-NEXT:    add a4, a4, a1
 ; CHECK-NEXT:    vl1r.v v18, (a7)
-; CHECK-NEXT:    add a7, a7, a2
-; CHECK-NEXT:    srli a2, a2, 1
-; CHECK-NEXT:    vl1r.v v9, (a3)
-; CHECK-NEXT:    add a3, a1, a1
-; CHECK-NEXT:    vl1r.v v17, (a5)
-; CHECK-NEXT:    add a5, a2, a2
+; CHECK-NEXT:    add a7, a7, a1
+; CHECK-NEXT:    srli a1, a1, 2
+; CHECK-NEXT:    vl1r.v v9, (a2)
+; CHECK-NEXT:    vl1r.v v17, (a4)
 ; CHECK-NEXT:    vl1r.v v11, (a0)
-; CHECK-NEXT:    vl1r.v v13, (a4)
+; CHECK-NEXT:    vl1r.v v13, (a5)
 ; CHECK-NEXT:    vl1r.v v19, (a7)
 ; CHECK-NEXT:    vl1r.v v15, (a6)
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -2390,12 +2373,12 @@ define <vscale x 96 x i1> @vector_interleave_nxv96i1_nxv16i1(<vscale x 16 x i1>
 ; CHECK-NEXT:    vmsne.vi v0, v12, 0
 ; CHECK-NEXT:    vmsne.vi v10, v18, 0
 ; CHECK-NEXT:    vmsne.vi v8, v14, 0
-; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v20, a1
 ; CHECK-NEXT:    vslideup.vx v0, v16, a1
-; CHECK-NEXT:    vsetvli zero, a5, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v9, a2
-; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v9, a3
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 12
@@ -2427,47 +2410,45 @@ define <vscale x 96 x i1> @vector_interleave_nxv96i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    vmv1r.v v17, v9
 ; ZVBB-NEXT:    vmv1r.v v0, v10
 ; ZVBB-NEXT:    vmerge.vim v24, v20, 1, v0
-; ZVBB-NEXT:    addi a5, sp, 16
+; ZVBB-NEXT:    addi a4, sp, 16
 ; ZVBB-NEXT:    vmv1r.v v18, v25
 ; ZVBB-NEXT:    vmv1r.v v0, v11
 ; ZVBB-NEXT:    vmerge.vim v26, v20, 1, v0
-; ZVBB-NEXT:    csrr a2, vlenb
+; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    vmv1r.v v19, v27
 ; ZVBB-NEXT:    vmv1r.v v0, v12
 ; ZVBB-NEXT:    vmerge.vim v10, v20, 1, v0
-; ZVBB-NEXT:    add a3, a0, a2
+; ZVBB-NEXT:    add a2, a0, a1
 ; ZVBB-NEXT:    vmv1r.v v20, v11
-; ZVBB-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a3, zero, e8, m1, ta, ma
 ; ZVBB-NEXT:    vsseg6e8.v v15, (a0)
 ; ZVBB-NEXT:    vmv1r.v v15, v22
-; ZVBB-NEXT:    add a4, a5, a2
+; ZVBB-NEXT:    add a5, a4, a1
 ; ZVBB-NEXT:    vmv1r.v v16, v8
-; ZVBB-NEXT:    srli a1, a2, 2
+; ZVBB-NEXT:    srli a3, a1, 1
 ; ZVBB-NEXT:    vmv1r.v v17, v24
-; ZVBB-NEXT:    add a6, a4, a2
+; ZVBB-NEXT:    add a6, a5, a1
 ; ZVBB-NEXT:    vmv1r.v v18, v26
-; ZVBB-NEXT:    add a7, a3, a2
+; ZVBB-NEXT:    add a7, a2, a1
 ; ZVBB-NEXT:    vmv1r.v v19, v10
-; ZVBB-NEXT:    vsseg6e8.v v14, (a5)
+; ZVBB-NEXT:    vsseg6e8.v v14, (a4)
 ; ZVBB-NEXT:    vl1r.v v8, (a0)
-; ZVBB-NEXT:    add a0, a6, a2
+; ZVBB-NEXT:    add a0, a6, a1
 ; ZVBB-NEXT:    vl1r.v v10, (a6)
-; ZVBB-NEXT:    add a6, a7, a2
-; ZVBB-NEXT:    vl1r.v v12, (a5)
-; ZVBB-NEXT:    add a5, a0, a2
+; ZVBB-NEXT:    add a6, a7, a1
+; ZVBB-NEXT:    vl1r.v v12, (a4)
+; ZVBB-NEXT:    add a4, a0, a1
 ; ZVBB-NEXT:    vl1r.v v14, (a7)
-; ZVBB-NEXT:    add a7, a6, a2
-; ZVBB-NEXT:    vl1r.v v16, (a5)
-; ZVBB-NEXT:    add a5, a5, a2
+; ZVBB-NEXT:    add a7, a6, a1
+; ZVBB-NEXT:    vl1r.v v16, (a4)
+; ZVBB-NEXT:    add a4, a4, a1
 ; ZVBB-NEXT:    vl1r.v v18, (a7)
-; ZVBB-NEXT:    add a7, a7, a2
-; ZVBB-NEXT:    srli a2, a2, 1
-; ZVBB-NEXT:    vl1r.v v9, (a3)
-; ZVBB-NEXT:    add a3, a1, a1
-; ZVBB-NEXT:    vl1r.v v17, (a5)
-; ZVBB-NEXT:    add a5, a2, a2
+; ZVBB-NEXT:    add a7, a7, a1
+; ZVBB-NEXT:    srli a1, a1, 2
+; ZVBB-NEXT:    vl1r.v v9, (a2)
+; ZVBB-NEXT:    vl1r.v v17, (a4)
 ; ZVBB-NEXT:    vl1r.v v11, (a0)
-; ZVBB-NEXT:    vl1r.v v13, (a4)
+; ZVBB-NEXT:    vl1r.v v13, (a5)
 ; ZVBB-NEXT:    vl1r.v v19, (a7)
 ; ZVBB-NEXT:    vl1r.v v15, (a6)
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
@@ -2477,12 +2458,12 @@ define <vscale x 96 x i1> @vector_interleave_nxv96i1_nxv16i1(<vscale x 16 x i1>
 ; ZVBB-NEXT:    vmsne.vi v0, v12, 0
 ; ZVBB-NEXT:    vmsne.vi v10, v18, 0
 ; ZVBB-NEXT:    vmsne.vi v8, v14, 0
-; ZVBB-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v20, a1
 ; ZVBB-NEXT:    vslideup.vx v0, v16, a1
-; ZVBB-NEXT:    vsetvli zero, a5, e8, m1, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v9, a2
-; ZVBB-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v9, a3
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    li a1, 12
@@ -3676,23 +3657,21 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; CHECK-NEXT:    sub sp, sp, a0
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmv.v.i v14, 0
-; CHECK-NEXT:    addi a4, sp, 16
+; CHECK-NEXT:    addi a3, sp, 16
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a1, a0, 3
 ; CHECK-NEXT:    sub a0, a1, a0
 ; CHECK-NEXT:    add a0, sp, a0
 ; CHECK-NEXT:    addi a0, a0, 16
-; CHECK-NEXT:    csrr a2, vlenb
+; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    vmerge.vim v16, v14, 1, v0
 ; CHECK-NEXT:    vmv1r.v v0, v8
 ; CHECK-NEXT:    vmerge.vim v22, v14, 1, v0
-; CHECK-NEXT:    add a3, a4, a2
-; CHECK-NEXT:    srli a1, a2, 2
-; CHECK-NEXT:    add a5, a0, a2
+; CHECK-NEXT:    add a2, a3, a1
 ; CHECK-NEXT:    vmv4r.v v24, v16
 ; CHECK-NEXT:    vmv1r.v v0, v9
 ; CHECK-NEXT:    vmerge.vim v18, v14, 1, v0
-; CHECK-NEXT:    add a6, a3, a2
+; CHECK-NEXT:    add a4, a2, a1
 ; CHECK-NEXT:    vmv1r.v v25, v22
 ; CHECK-NEXT:    vmv1r.v v0, v10
 ; CHECK-NEXT:    vmerge.vim v8, v14, 1, v0
@@ -3704,41 +3683,41 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; CHECK-NEXT:    vmerge.vim v10, v14, 1, v0
 ; CHECK-NEXT:    vmv1r.v v28, v20
 ; CHECK-NEXT:    vmv1r.v v18, v23
-; CHECK-NEXT:    add a7, a6, a2
+; CHECK-NEXT:    add a5, a4, a1
 ; CHECK-NEXT:    vmv1r.v v29, v10
 ; CHECK-NEXT:    vmv1r.v v20, v9
 ; CHECK-NEXT:    vmv1r.v v0, v13
 ; CHECK-NEXT:    vmerge.vim v30, v14, 1, v0
 ; CHECK-NEXT:    vmv1r.v v22, v11
-; CHECK-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vsseg7e8.v v24, (a4)
+; CHECK-NEXT:    vsetvli a6, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vsseg7e8.v v24, (a3)
 ; CHECK-NEXT:    vmv1r.v v23, v31
 ; CHECK-NEXT:    vsseg7e8.v v17, (a0)
-; CHECK-NEXT:    vl1r.v v8, (a6)
-; CHECK-NEXT:    add a6, a7, a2
-; CHECK-NEXT:    vl1r.v v10, (a4)
-; CHECK-NEXT:    add a4, a6, a2
-; CHECK-NEXT:    vl1r.v v12, (a6)
-; CHECK-NEXT:    add a6, a4, a2
-; CHECK-NEXT:    vl1r.v v14, (a6)
-; CHECK-NEXT:    add a6, a5, a2
-; CHECK-NEXT:    vl1r.v v16, (a5)
-; CHECK-NEXT:    add a5, a6, a2
-; CHECK-NEXT:    vl1r.v v18, (a5)
-; CHECK-NEXT:    add a5, a5, a2
-; CHECK-NEXT:    vl1r.v v9, (a7)
-; CHECK-NEXT:    add a7, a5, a2
-; CHECK-NEXT:    vl1r.v v20, (a7)
-; CHECK-NEXT:    add a7, a7, a2
-; CHECK-NEXT:    srli a2, a2, 1
-; CHECK-NEXT:    vl1r.v v11, (a3)
-; CHECK-NEXT:    add a3, a1, a1
-; CHECK-NEXT:    vl1r.v v13, (a4)
-; CHECK-NEXT:    add a4, a2, a2
+; CHECK-NEXT:    vl1r.v v8, (a4)
+; CHECK-NEXT:    add a4, a5, a1
+; CHECK-NEXT:    vl1r.v v10, (a3)
+; CHECK-NEXT:    add a6, a4, a1
+; CHECK-NEXT:    vl1r.v v12, (a4)
+; CHECK-NEXT:    add a3, a6, a1
+; CHECK-NEXT:    vl1r.v v14, (a3)
+; CHECK-NEXT:    srli a3, a1, 1
+; CHECK-NEXT:    vl1r.v v9, (a5)
+; CHECK-NEXT:    add a4, a0, a1
+; CHECK-NEXT:    vl1r.v v16, (a4)
+; CHECK-NEXT:    add a4, a4, a1
+; CHECK-NEXT:    vl1r.v v11, (a2)
+; CHECK-NEXT:    add a2, a4, a1
+; CHECK-NEXT:    vl1r.v v18, (a2)
+; CHECK-NEXT:    add a2, a2, a1
+; CHECK-NEXT:    vl1r.v v13, (a6)
+; CHECK-NEXT:    add a5, a2, a1
+; CHECK-NEXT:    vl1r.v v20, (a5)
+; CHECK-NEXT:    add a5, a5, a1
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    vl1r.v v15, (a0)
-; CHECK-NEXT:    vl1r.v v19, (a5)
-; CHECK-NEXT:    vl1r.v v17, (a6)
-; CHECK-NEXT:    vl1r.v v21, (a7)
+; CHECK-NEXT:    vl1r.v v19, (a2)
+; CHECK-NEXT:    vl1r.v v17, (a4)
+; CHECK-NEXT:    vl1r.v v21, (a5)
 ; CHECK-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmsne.vi v22, v8, 0
 ; CHECK-NEXT:    vmsne.vi v0, v10, 0
@@ -3747,13 +3726,13 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; CHECK-NEXT:    vmsne.vi v11, v18, 0
 ; CHECK-NEXT:    vmsne.vi v8, v16, 0
 ; CHECK-NEXT:    vmsne.vi v12, v20, 0
-; CHECK-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v0, v22, a1
 ; CHECK-NEXT:    vslideup.vx v9, v10, a1
 ; CHECK-NEXT:    vslideup.vx v8, v11, a1
-; CHECK-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vx v0, v9, a2
-; CHECK-NEXT:    vslideup.vx v8, v12, a2
+; CHECK-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vslideup.vx v0, v9, a3
+; CHECK-NEXT:    vslideup.vx v8, v12, a3
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    li a1, 14
 ; CHECK-NEXT:    mul a0, a0, a1
@@ -3770,23 +3749,21 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; ZVBB-NEXT:    sub sp, sp, a0
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; ZVBB-NEXT:    vmv.v.i v14, 0
-; ZVBB-NEXT:    addi a4, sp, 16
+; ZVBB-NEXT:    addi a3, sp, 16
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a1, a0, 3
 ; ZVBB-NEXT:    sub a0, a1, a0
 ; ZVBB-NEXT:    add a0, sp, a0
 ; ZVBB-NEXT:    addi a0, a0, 16
-; ZVBB-NEXT:    csrr a2, vlenb
+; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    vmerge.vim v16, v14, 1, v0
 ; ZVBB-NEXT:    vmv1r.v v0, v8
 ; ZVBB-NEXT:    vmerge.vim v22, v14, 1, v0
-; ZVBB-NEXT:    add a3, a4, a2
-; ZVBB-NEXT:    srli a1, a2, 2
-; ZVBB-NEXT:    add a5, a0, a2
+; ZVBB-NEXT:    add a2, a3, a1
 ; ZVBB-NEXT:    vmv4r.v v24, v16
 ; ZVBB-NEXT:    vmv1r.v v0, v9
 ; ZVBB-NEXT:    vmerge.vim v18, v14, 1, v0
-; ZVBB-NEXT:    add a6, a3, a2
+; ZVBB-NEXT:    add a4, a2, a1
 ; ZVBB-NEXT:    vmv1r.v v25, v22
 ; ZVBB-NEXT:    vmv1r.v v0, v10
 ; ZVBB-NEXT:    vmerge.vim v8, v14, 1, v0
@@ -3798,41 +3775,41 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; ZVBB-NEXT:    vmerge.vim v10, v14, 1, v0
 ; ZVBB-NEXT:    vmv1r.v v28, v20
 ; ZVBB-NEXT:    vmv1r.v v18, v23
-; ZVBB-NEXT:    add a7, a6, a2
+; ZVBB-NEXT:    add a5, a4, a1
 ; ZVBB-NEXT:    vmv1r.v v29, v10
 ; ZVBB-NEXT:    vmv1r.v v20, v9
 ; ZVBB-NEXT:    vmv1r.v v0, v13
 ; ZVBB-NEXT:    vmerge.vim v30, v14, 1, v0
 ; ZVBB-NEXT:    vmv1r.v v22, v11
-; ZVBB-NEXT:    vsetvli t0, zero, e8, m1, ta, ma
-; ZVBB-NEXT:    vsseg7e8.v v24, (a4)
+; ZVBB-NEXT:    vsetvli a6, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vsseg7e8.v v24, (a3)
 ; ZVBB-NEXT:    vmv1r.v v23, v31
 ; ZVBB-NEXT:    vsseg7e8.v v17, (a0)
-; ZVBB-NEXT:    vl1r.v v8, (a6)
-; ZVBB-NEXT:    add a6, a7, a2
-; ZVBB-NEXT:    vl1r.v v10, (a4)
-; ZVBB-NEXT:    add a4, a6, a2
-; ZVBB-NEXT:    vl1r.v v12, (a6)
-; ZVBB-NEXT:    add a6, a4, a2
-; ZVBB-NEXT:    vl1r.v v14, (a6)
-; ZVBB-NEXT:    add a6, a5, a2
-; ZVBB-NEXT:    vl1r.v v16, (a5)
-; ZVBB-NEXT:    add a5, a6, a2
-; ZVBB-NEXT:    vl1r.v v18, (a5)
-; ZVBB-NEXT:    add a5, a5, a2
-; ZVBB-NEXT:    vl1r.v v9, (a7)
-; ZVBB-NEXT:    add a7, a5, a2
-; ZVBB-NEXT:    vl1r.v v20, (a7)
-; ZVBB-NEXT:    add a7, a7, a2
-; ZVBB-NEXT:    srli a2, a2, 1
-; ZVBB-NEXT:    vl1r.v v11, (a3)
-; ZVBB-NEXT:    add a3, a1, a1
-; ZVBB-NEXT:    vl1r.v v13, (a4)
-; ZVBB-NEXT:    add a4, a2, a2
+; ZVBB-NEXT:    vl1r.v v8, (a4)
+; ZVBB-NEXT:    add a4, a5, a1
+; ZVBB-NEXT:    vl1r.v v10, (a3)
+; ZVBB-NEXT:    add a6, a4, a1
+; ZVBB-NEXT:    vl1r.v v12, (a4)
+; ZVBB-NEXT:    add a3, a6, a1
+; ZVBB-NEXT:    vl1r.v v14, (a3)
+; ZVBB-NEXT:    srli a3, a1, 1
+; ZVBB-NEXT:    vl1r.v v9, (a5)
+; ZVBB-NEXT:    add a4, a0, a1
+; ZVBB-NEXT:    vl1r.v v16, (a4)
+; ZVBB-NEXT:    add a4, a4, a1
+; ZVBB-NEXT:    vl1r.v v11, (a2)
+; ZVBB-NEXT:    add a2, a4, a1
+; ZVBB-NEXT:    vl1r.v v18, (a2)
+; ZVBB-NEXT:    add a2, a2, a1
+; ZVBB-NEXT:    vl1r.v v13, (a6)
+; ZVBB-NEXT:    add a5, a2, a1
+; ZVBB-NEXT:    vl1r.v v20, (a5)
+; ZVBB-NEXT:    add a5, a5, a1
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    vl1r.v v15, (a0)
-; ZVBB-NEXT:    vl1r.v v19, (a5)
-; ZVBB-NEXT:    vl1r.v v17, (a6)
-; ZVBB-NEXT:    vl1r.v v21, (a7)
+; ZVBB-NEXT:    vl1r.v v19, (a2)
+; ZVBB-NEXT:    vl1r.v v17, (a4)
+; ZVBB-NEXT:    vl1r.v v21, (a5)
 ; ZVBB-NEXT:    vsetvli a0, zero, e8, m2, ta, ma
 ; ZVBB-NEXT:    vmsne.vi v22, v8, 0
 ; ZVBB-NEXT:    vmsne.vi v0, v10, 0
@@ -3841,13 +3818,13 @@ define <vscale x 112 x i1> @vector_interleave_nxv112i1_nxv16i1(<vscale x 16 x i1
 ; ZVBB-NEXT:    vmsne.vi v11, v18, 0
 ; ZVBB-NEXT:    vmsne.vi v8, v16, 0
 ; ZVBB-NEXT:    vmsne.vi v12, v20, 0
-; ZVBB-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e8, mf2, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v0, v22, a1
 ; ZVBB-NEXT:    vslideup.vx v9, v10, a1
 ; ZVBB-NEXT:    vslideup.vx v8, v11, a1
-; ZVBB-NEXT:    vsetvli zero, a4, e8, m1, ta, ma
-; ZVBB-NEXT:    vslideup.vx v0, v9, a2
-; ZVBB-NEXT:    vslideup.vx v8, v12, a2
+; ZVBB-NEXT:    vsetvli a0, zero, e8, m1, ta, ma
+; ZVBB-NEXT:    vslideup.vx v0, v9, a3
+; ZVBB-NEXT:    vslideup.vx v8, v12, a3
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    li a1, 14
 ; ZVBB-NEXT:    mul a0, a0, a1
@@ -5569,54 +5546,52 @@ define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv16i1(<vscale x 16 x i1
 ; CHECK-NEXT:    add a6, a4, a0
 ; CHECK-NEXT:    add a7, a5, a0
 ; CHECK-NEXT:    add t0, a6, a0
-; CHECK-NEXT:    add t1, a7, a0
-; CHECK-NEXT:    add t2, t0, a0
 ; CHECK-NEXT:    vmv1r.v v20, v9
-; CHECK-NEXT:    add t3, t1, a0
+; CHECK-NEXT:    add t1, a7, a0
 ; CHECK-NEXT:    vmv1r.v v22, v11
 ; CHECK-NEXT:    vsseg8e8.v v16, (a1)
-; CHECK-NEXT:    vl1r.v v10, (t1)
-; CHECK-NEXT:    add t1, t2, a0
-; CHECK-NEXT:    vl1r.v v12, (a5)
-; CHECK-NEXT:    add a5, t3, a0
+; CHECK-NEXT:    vl1r.v v8, (a5)
+; CHECK-NEXT:    add a5, t0, a0
+; CHECK-NEXT:    vl1r.v v12, (t1)
+; CHECK-NEXT:    add t1, t1, a0
 ; CHECK-NEXT:    vl1r.v v14, (a2)
-; CHECK-NEXT:    add a2, t1, a0
+; CHECK-NEXT:    add a2, a5, a0
+; CHECK-NEXT:    vl1r.v v10, (a5)
+; CHECK-NEXT:    add a5, t1, a0
 ; CHECK-NEXT:    vl1r.v v16, (a5)
 ; CHECK-NEXT:    add a5, a5, a0
-; CHECK-NEXT:    vl1r.v v8, (a2)
-; CHECK-NEXT:    add a2, a2, a0
-; CHECK-NEXT:    vl1r.v v18, (t2)
 ; CHECK-NEXT:    vl1r.v v17, (a5)
-; CHECK-NEXT:    vl1r.v v11, (t3)
-; CHECK-NEXT:    vl1r.v v13, (a7)
+; CHECK-NEXT:    add a5, a2, a0
+; CHECK-NEXT:    vl1r.v v18, (a5)
+; CHECK-NEXT:    add a5, a5, a0
+; CHECK-NEXT:    vl1r.v v13, (t1)
+; CHECK-NEXT:    vl1r.v v9, (a7)
 ; CHECK-NEXT:    vl1r.v v15, (a3)
 ; CHECK-NEXT:    vsetvli a3, zero, e8, m2, ta, ma
 ; CHECK-NEXT:    vmsne.vi v20, v16, 0
-; CHECK-NEXT:    vmsne.vi v16, v10, 0
-; CHECK-NEXT:    vl1r.v v10, (a6)
-; CHECK-NEXT:    vmsne.vi v17, v12, 0
+; CHECK-NEXT:    vmsne.vi v16, v12, 0
+; CHECK-NEXT:    vl1r.v v12, (a6)
+; CHECK-NEXT:    vmsne.vi v17, v8, 0
 ; CHECK-NEXT:    vmsne.vi v0, v14, 0
-; CHECK-NEXT:    vl1r.v v12, (a1)
-; CHECK-NEXT:    vl1r.v v9, (a2)
-; CHECK-NEXT:    vl1r.v v19, (t1)
-; CHECK-NEXT:    vl1r.v v11, (t0)
-; CHECK-NEXT:    vl1r.v v13, (a4)
-; CHECK-NEXT:    vmsne.vi v14, v8, 0
+; CHECK-NEXT:    vl1r.v v14, (a1)
+; CHECK-NEXT:    vl1r.v v19, (a5)
+; CHECK-NEXT:    vl1r.v v11, (a2)
+; CHECK-NEXT:    vl1r.v v13, (t0)
+; CHECK-NEXT:    vl1r.v v15, (a4)
 ; CHECK-NEXT:    vmsne.vi v9, v18, 0
-; CHECK-NEXT:    vmsne.vi v15, v10, 0
-; CHECK-NEXT:    vmsne.vi v8, v12, 0
+; CHECK-NEXT:    vmsne.vi v18, v10, 0
+; CHECK-NEXT:    vmsne.vi v10, v12, 0
+; CHECK-NEXT:    vmsne.vi v8, v14, 0
 ; CHECK-NEXT:    srli a1, a0, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v16, v20, a1
 ; CHECK-NEXT:    vslideup.vx v0, v17, a1
-; CHECK-NEXT:    vslideup.vx v9, v14, a1
-; CHECK-NEXT:    vslideup.vx v8, v15, a1
+; CHECK-NEXT:    vslideup.vx v18, v9, a1
+; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    srli a0, a0, 1
-; CHECK-NEXT:    add a1, a0, a0
-; CHECK-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v0, v16, a0
-; CHECK-NEXT:    vslideup.vx v8, v9, a0
+; CHECK-NEXT:    vslideup.vx v8, v18, a0
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 4
 ; CHECK-NEXT:    add sp, sp, a0
@@ -5670,54 +5645,52 @@ define <vscale x 128 x i1> @vector_interleave_nxv128i1_nxv16i1(<vscale x 16 x i1
 ; ZVBB-NEXT:    add a6, a4, a0
 ; ZVBB-NEXT:    add a7, a5, a0
 ; ZVBB-NEXT:    add t0, a6, a0
-; ZVBB-NEXT:    add t1, a7, a0
-; ZVBB-NEXT:    add t2, t0, a0
 ; ZVBB-NEXT:    vmv1r.v v20, v9
-; ZVBB-NEXT:    add t3, t1, a0
+; ZVBB-NEXT:    add t1, a7, a0
 ; ZVBB-NEXT:    vmv1r.v v22, v11
 ; ZVBB-NEXT:    vsseg8e8.v v16, (a1)
-; ZVBB-NEXT:    vl1r.v v10, (t1)
-; ZVBB-NEXT:    add t1, t2, a0
-; ZVBB-NEXT:    vl1r.v v12, (a5)
-; ZVBB-NEXT:    add a5, t3, a0
+; ZVBB-NEXT:    vl1r.v v8, (a5)
+; ZVBB-NEXT:    add a5, t0, a0
+; ZVBB-NEXT:    vl1r.v v12, (t1)
+; ZVBB-NEXT:    add t1, t1, a0
 ; ZVBB-NEXT:    vl1r.v v14, (a2)
-; ZVBB-NEXT:    add a2, t1, a0
+; ZVBB-NEXT:    add a2, a5, a0
+; ZVBB-NEXT:    vl1r.v v10, (a5)
+; ZVBB-NEXT:    add a5, t1, a0
 ; ZVBB-NEXT:    vl1r.v v16, (a5)
 ; ZVBB-NEXT:    add a5, a5, a0
-; ZVBB-NEXT:    vl1r.v v8, (a2)
-; ZVBB-NEXT:    add a2, a2, a0
-; ZVBB-NEXT:    vl1r.v v18, (t2)
 ; ZVBB-NEXT:    vl1r.v v17, (a5)
-; ZVBB-NEXT:    vl1r.v v11, (t3)
-; ZVBB-NEXT:    vl1r.v v13, (a7)
+; ZVBB-NEXT:    add a5, a2, a0
+; ZVBB-NEXT:    vl1r.v v18, (a5)
+; ZVBB-NEXT:    add a5, a5, a0
+; ZVBB-NEXT:    vl1r.v v13, (t1)
+; ZVBB-NEXT:    vl1r.v v9, (a7)
 ; ZVBB-NEXT:    vl1r.v v15, (a3)
 ; ZVBB-NEXT:    vsetvli a3, zero, e8, m2, ta, ma
 ; ZVBB-NEXT:    vmsne.vi v20, v16, 0
-; ZVBB-NEXT:    vmsne.vi v16, v10, 0
-; ZVBB-NEXT:    vl1r.v v10, (a6)
-; ZVBB-NEXT:    vmsne.vi v17, v12, 0
+; ZVBB-NEXT:    vmsne.vi v16, v12, 0
+; ZVBB-NEXT:    vl1r.v v12, (a6)
+; ZVBB-NEXT:    vmsne.vi v17, v8, 0
 ; ZVBB-NEXT:    vmsne.vi v0, v14, 0
-; ZVBB-NEXT:    vl1r.v v12, (a1)
-; ZVBB-NEXT:    vl1r.v v9, (a2)
-; ZVBB-NEXT:    vl1r.v v19, (t1)
-; ZVBB-NEXT:    vl1r.v v11, (t0)
-; ZVBB-NEXT:    vl1r.v v13, (a4)
-; ZVBB-NEXT:    vmsne.vi v14, v8, 0
+; ZVBB-NEXT:    vl1r.v v14, (a1)
+; ZVBB-NEXT:    vl1r.v v19, (a5)
+; ZVBB-NEXT:    vl1r.v v11, (a2)
+; ZVBB-NEXT:    vl1r.v v13, (t0)
+; ZVBB-NEXT:    vl1r.v v15, (a4)
 ; ZVBB-NEXT:    vmsne.vi v9, v18, 0
-; ZVBB-NEXT:    vmsne.vi v15, v10, 0
-; ZVBB-NEXT:    vmsne.vi v8, v12, 0
+; ZVBB-NEXT:    vmsne.vi v18, v10, 0
+; ZVBB-NEXT:    vmsne.vi v10, v12, 0
+; ZVBB-NEXT:    vmsne.vi v8, v14, 0
 ; ZVBB-NEXT:    srli a1, a0, 2
-; ZVBB-NEXT:    add a2, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v16, v20, a1
 ; ZVBB-NEXT:    vslideup.vx v0, v17, a1
-; ZVBB-NEXT:    vslideup.vx v9, v14, a1
-; ZVBB-NEXT:    vslideup.vx v8, v15, a1
+; ZVBB-NEXT:    vslideup.vx v18, v9, a1
+; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    srli a0, a0, 1
-; ZVBB-NEXT:    add a1, a0, a0
-; ZVBB-NEXT:    vsetvli zero, a1, e8, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v0, v16, a0
-; ZVBB-NEXT:    vslideup.vx v8, v9, a0
+; ZVBB-NEXT:    vslideup.vx v8, v18, a0
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 4
 ; ZVBB-NEXT:    add sp, sp, a0
@@ -6294,14 +6267,12 @@ define <vscale x 4 x bfloat> @vector_interleave_nxv4bf16_nxv2bf16(<vscale x 2 x
 ; V-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; V-NEXT:    vwaddu.vv v10, v8, v9
 ; V-NEXT:    li a0, -1
-; V-NEXT:    csrr a1, vlenb
 ; V-NEXT:    vwmaccu.vx v10, a0, v9
-; V-NEXT:    srli a1, a1, 2
-; V-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; V-NEXT:    vslidedown.vx v8, v10, a1
-; V-NEXT:    add a0, a1, a1
-; V-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; V-NEXT:    vslideup.vx v10, v8, a1
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    srli a0, a0, 2
+; V-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; V-NEXT:    vslidedown.vx v8, v10, a0
+; V-NEXT:    vslideup.vx v10, v8, a0
 ; V-NEXT:    vmv.v.v v8, v10
 ; V-NEXT:    ret
 ;
@@ -6314,8 +6285,6 @@ define <vscale x 4 x bfloat> @vector_interleave_nxv4bf16_nxv2bf16(<vscale x 2 x
 ; ZVBB-NEXT:    srli a0, a0, 2
 ; ZVBB-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslidedown.vx v8, v10, a0
-; ZVBB-NEXT:    add a1, a0, a0
-; ZVBB-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a0
 ; ZVBB-NEXT:    vmv.v.v v8, v10
 ; ZVBB-NEXT:    ret
@@ -6327,8 +6296,7 @@ define <vscale x 4 x bfloat> @vector_interleave_nxv4bf16_nxv2bf16(<vscale x 2 x
 ; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v9
 ; ZIP-NEXT:    csrr a0, vlenb
 ; ZIP-NEXT:    srli a0, a0, 2
-; ZIP-NEXT:    add a1, a0, a0
-; ZIP-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; ZIP-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZIP-NEXT:    vslideup.vx v10, v11, a0
 ; ZIP-NEXT:    vmv.v.v v8, v10
 ; ZIP-NEXT:    ret
@@ -6374,14 +6342,12 @@ define <vscale x 4 x half> @vector_interleave_nxv4f16_nxv2f16(<vscale x 2 x half
 ; V-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
 ; V-NEXT:    vwaddu.vv v10, v8, v9
 ; V-NEXT:    li a0, -1
-; V-NEXT:    csrr a1, vlenb
 ; V-NEXT:    vwmaccu.vx v10, a0, v9
-; V-NEXT:    srli a1, a1, 2
-; V-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
-; V-NEXT:    vslidedown.vx v8, v10, a1
-; V-NEXT:    add a0, a1, a1
-; V-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
-; V-NEXT:    vslideup.vx v10, v8, a1
+; V-NEXT:    csrr a0, vlenb
+; V-NEXT:    srli a0, a0, 2
+; V-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; V-NEXT:    vslidedown.vx v8, v10, a0
+; V-NEXT:    vslideup.vx v10, v8, a0
 ; V-NEXT:    vmv.v.v v8, v10
 ; V-NEXT:    ret
 ;
@@ -6394,8 +6360,6 @@ define <vscale x 4 x half> @vector_interleave_nxv4f16_nxv2f16(<vscale x 2 x half
 ; ZVBB-NEXT:    srli a0, a0, 2
 ; ZVBB-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslidedown.vx v8, v10, a0
-; ZVBB-NEXT:    add a1, a0, a0
-; ZVBB-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a0
 ; ZVBB-NEXT:    vmv.v.v v8, v10
 ; ZVBB-NEXT:    ret
@@ -6407,8 +6371,7 @@ define <vscale x 4 x half> @vector_interleave_nxv4f16_nxv2f16(<vscale x 2 x half
 ; ZIP-NEXT:    ri.vzip2a.vv v10, v8, v9
 ; ZIP-NEXT:    csrr a0, vlenb
 ; ZIP-NEXT:    srli a0, a0, 2
-; ZIP-NEXT:    add a1, a0, a0
-; ZIP-NEXT:    vsetvli zero, a1, e16, m1, ta, ma
+; ZIP-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
 ; ZIP-NEXT:    vslideup.vx v10, v11, a0
 ; ZIP-NEXT:    vmv.v.v v8, v10
 ; ZIP-NEXT:    ret
@@ -6807,8 +6770,7 @@ define <vscale x 6 x half> @vector_interleave_nxv6f16_nxv2f16(<vscale x 2 x half
 ; CHECK-NEXT:    vle16.v v9, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a1
 ; CHECK-NEXT:    add a2, a3, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -6834,8 +6796,7 @@ define <vscale x 6 x half> @vector_interleave_nxv6f16_nxv2f16(<vscale x 2 x half
 ; ZVBB-NEXT:    vle16.v v9, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v9, a1
 ; ZVBB-NEXT:    add a2, a3, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -6967,8 +6928,7 @@ define <vscale x 6 x bfloat> @vector_interleave_nxv6bf16_nxv2bf16(<vscale x 2 x
 ; CHECK-NEXT:    vle16.v v9, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a1
 ; CHECK-NEXT:    add a2, a3, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -6994,8 +6954,7 @@ define <vscale x 6 x bfloat> @vector_interleave_nxv6bf16_nxv2bf16(<vscale x 2 x
 ; ZVBB-NEXT:    vle16.v v9, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v9, a1
 ; ZVBB-NEXT:    add a2, a3, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -7127,8 +7086,7 @@ define <vscale x 3 x float> @vector_interleave_nxv3f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    vle32.v v9, (a3)
 ; CHECK-NEXT:    vle32.v v8, (a0)
 ; CHECK-NEXT:    srli a1, a1, 3
-; CHECK-NEXT:    add a0, a1, a1
-; CHECK-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v9, a1
 ; CHECK-NEXT:    add a2, a3, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
@@ -7154,8 +7112,7 @@ define <vscale x 3 x float> @vector_interleave_nxv3f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    vle32.v v9, (a3)
 ; ZVBB-NEXT:    vle32.v v8, (a0)
 ; ZVBB-NEXT:    srli a1, a1, 3
-; ZVBB-NEXT:    add a0, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a0, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v9, a1
 ; ZVBB-NEXT:    add a2, a3, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
@@ -7391,13 +7348,12 @@ define <vscale x 8 x half> @vector_interleave_nxv8f16_nxv2f16(<vscale x 2 x half
 ; CHECK-NEXT:    vle16.v v9, (a4)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v10, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
@@ -7422,13 +7378,12 @@ define <vscale x 8 x half> @vector_interleave_nxv8f16_nxv2f16(<vscale x 2 x half
 ; ZVBB-NEXT:    vle16.v v9, (a4)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v10, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 1
@@ -7559,13 +7514,12 @@ define <vscale x 8 x bfloat> @vector_interleave_nxv8bf16_nxv2bf16(<vscale x 2 x
 ; CHECK-NEXT:    vle16.v v9, (a4)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v10, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
@@ -7590,13 +7544,12 @@ define <vscale x 8 x bfloat> @vector_interleave_nxv8bf16_nxv2bf16(<vscale x 2 x
 ; ZVBB-NEXT:    vle16.v v9, (a4)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v10, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 1
@@ -7727,13 +7680,12 @@ define <vscale x 4 x float> @vector_interleave_nxv4f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    vle32.v v9, (a4)
 ; CHECK-NEXT:    vle32.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 3
-; CHECK-NEXT:    add a2, a1, a1
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v10, (a3)
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 1
@@ -7758,13 +7710,12 @@ define <vscale x 4 x float> @vector_interleave_nxv4f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    vle32.v v9, (a4)
 ; ZVBB-NEXT:    vle32.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 3
-; ZVBB-NEXT:    add a2, a1, a1
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v10, (a3)
 ; ZVBB-NEXT:    vle32.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 1
@@ -7998,13 +7949,12 @@ define <vscale x 10 x half> @vector_interleave_nxv10f16_nxv2f16(<vscale x 2 x ha
 ; CHECK-NEXT:    vle16.v v8, (a5)
 ; CHECK-NEXT:    vle16.v v9, (a4)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a4, a1, a1
 ; CHECK-NEXT:    vle16.v v10, (a3)
-; CHECK-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
 ; CHECK-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    add a2, a5, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -8034,13 +7984,12 @@ define <vscale x 10 x half> @vector_interleave_nxv10f16_nxv2f16(<vscale x 2 x ha
 ; ZVBB-NEXT:    vle16.v v8, (a5)
 ; ZVBB-NEXT:    vle16.v v9, (a4)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a4, a1, a1
 ; ZVBB-NEXT:    vle16.v v10, (a3)
-; ZVBB-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
 ; ZVBB-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    add a2, a5, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -8466,13 +8415,12 @@ define <vscale x 10 x bfloat> @vector_interleave_nxv10bf16_nxv2bf16(<vscale x 2
 ; CHECK-NEXT:    vle16.v v8, (a5)
 ; CHECK-NEXT:    vle16.v v9, (a4)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a4, a1, a1
 ; CHECK-NEXT:    vle16.v v10, (a3)
-; CHECK-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
 ; CHECK-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    add a2, a5, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -8502,13 +8450,12 @@ define <vscale x 10 x bfloat> @vector_interleave_nxv10bf16_nxv2bf16(<vscale x 2
 ; ZVBB-NEXT:    vle16.v v8, (a5)
 ; ZVBB-NEXT:    vle16.v v9, (a4)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a4, a1, a1
 ; ZVBB-NEXT:    vle16.v v10, (a3)
-; ZVBB-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a3, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
 ; ZVBB-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a4, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    add a2, a5, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e16, mf2, ta, ma
@@ -8934,13 +8881,12 @@ define <vscale x 5 x float> @vector_interleave_nxv5f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    vle32.v v8, (a5)
 ; CHECK-NEXT:    vle32.v v9, (a4)
 ; CHECK-NEXT:    srli a1, a1, 3
-; CHECK-NEXT:    add a4, a1, a1
 ; CHECK-NEXT:    vle32.v v10, (a3)
-; CHECK-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
 ; CHECK-NEXT:    vsetvli a3, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v10, a1
 ; CHECK-NEXT:    add a2, a5, a2
 ; CHECK-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
@@ -8970,13 +8916,12 @@ define <vscale x 5 x float> @vector_interleave_nxv5f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    vle32.v v8, (a5)
 ; ZVBB-NEXT:    vle32.v v9, (a4)
 ; ZVBB-NEXT:    srli a1, a1, 3
-; ZVBB-NEXT:    add a4, a1, a1
 ; ZVBB-NEXT:    vle32.v v10, (a3)
-; ZVBB-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
 ; ZVBB-NEXT:    vsetvli a3, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v10, a1
 ; ZVBB-NEXT:    add a2, a5, a2
 ; ZVBB-NEXT:    vsetvli a0, zero, e32, mf2, ta, ma
@@ -9796,18 +9741,17 @@ define <vscale x 12 x half> @vector_interleave_nxv12f16_nxv2f16(<vscale x 2 x ha
 ; CHECK-NEXT:    vle16.v v10, (a6)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle16.v v11, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
-; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v11, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -9836,18 +9780,17 @@ define <vscale x 12 x half> @vector_interleave_nxv12f16_nxv2f16(<vscale x 2 x ha
 ; ZVBB-NEXT:    vle16.v v10, (a6)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle16.v v11, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v11, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v11, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v11, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a1, a0, 1
@@ -10311,18 +10254,17 @@ define <vscale x 12 x bfloat> @vector_interleave_nxv12bf16_nxv2bf16(<vscale x 2
 ; CHECK-NEXT:    vle16.v v10, (a6)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle16.v v11, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
-; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v11, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -10351,18 +10293,17 @@ define <vscale x 12 x bfloat> @vector_interleave_nxv12bf16_nxv2bf16(<vscale x 2
 ; ZVBB-NEXT:    vle16.v v10, (a6)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle16.v v11, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v11, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v11, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v11, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a1, a0, 1
@@ -10826,18 +10767,17 @@ define <vscale x 6 x float> @vector_interleave_nxv6f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    vle32.v v10, (a6)
 ; CHECK-NEXT:    vle32.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 3
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle32.v v11, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
-; CHECK-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v11, a1
-; CHECK-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v11, (a3)
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v11, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a1, a0, 1
@@ -10866,18 +10806,17 @@ define <vscale x 6 x float> @vector_interleave_nxv6f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    vle32.v v10, (a6)
 ; ZVBB-NEXT:    vle32.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 3
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle32.v v11, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v11, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v11, (a3)
 ; ZVBB-NEXT:    vle32.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v11, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a1, a0, 1
@@ -11761,7 +11700,6 @@ define <vscale x 14 x half> @vector_interleave_nxv14f16_nxv2f16(<vscale x 2 x ha
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 1
-; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    add a3, a0, a2
 ; CHECK-NEXT:    add a4, a3, a2
 ; CHECK-NEXT:    add a5, a4, a2
@@ -11771,20 +11709,20 @@ define <vscale x 14 x half> @vector_interleave_nxv14f16_nxv2f16(<vscale x 2 x ha
 ; CHECK-NEXT:    add a7, a6, a2
 ; CHECK-NEXT:    vle16.v v8, (a7)
 ; CHECK-NEXT:    vle16.v v10, (a6)
-; CHECK-NEXT:    add a6, a1, a1
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    add a2, a7, a2
 ; CHECK-NEXT:    vle16.v v12, (a5)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a5, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v11, (a2)
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v12, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v12, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -11801,7 +11739,6 @@ define <vscale x 14 x half> @vector_interleave_nxv14f16_nxv2f16(<vscale x 2 x ha
 ; ZVBB-NEXT:    addi a0, sp, 16
 ; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    srli a2, a1, 1
-; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    add a3, a0, a2
 ; ZVBB-NEXT:    add a4, a3, a2
 ; ZVBB-NEXT:    add a5, a4, a2
@@ -11811,20 +11748,20 @@ define <vscale x 14 x half> @vector_interleave_nxv14f16_nxv2f16(<vscale x 2 x ha
 ; ZVBB-NEXT:    add a7, a6, a2
 ; ZVBB-NEXT:    vle16.v v8, (a7)
 ; ZVBB-NEXT:    vle16.v v10, (a6)
-; ZVBB-NEXT:    add a6, a1, a1
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    add a2, a7, a2
 ; ZVBB-NEXT:    vle16.v v12, (a5)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a5, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
 ; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v11, (a2)
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v12, a1
 ; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v12, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
@@ -12325,7 +12262,6 @@ define <vscale x 14 x bfloat> @vector_interleave_nxv14bf16_nxv2bf16(<vscale x 2
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 1
-; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    add a3, a0, a2
 ; CHECK-NEXT:    add a4, a3, a2
 ; CHECK-NEXT:    add a5, a4, a2
@@ -12335,20 +12271,20 @@ define <vscale x 14 x bfloat> @vector_interleave_nxv14bf16_nxv2bf16(<vscale x 2
 ; CHECK-NEXT:    add a7, a6, a2
 ; CHECK-NEXT:    vle16.v v8, (a7)
 ; CHECK-NEXT:    vle16.v v10, (a6)
-; CHECK-NEXT:    add a6, a1, a1
+; CHECK-NEXT:    srli a1, a1, 2
 ; CHECK-NEXT:    add a2, a7, a2
 ; CHECK-NEXT:    vle16.v v12, (a5)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a5, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v11, (a2)
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v12, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v12, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -12365,7 +12301,6 @@ define <vscale x 14 x bfloat> @vector_interleave_nxv14bf16_nxv2bf16(<vscale x 2
 ; ZVBB-NEXT:    addi a0, sp, 16
 ; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    srli a2, a1, 1
-; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    add a3, a0, a2
 ; ZVBB-NEXT:    add a4, a3, a2
 ; ZVBB-NEXT:    add a5, a4, a2
@@ -12375,20 +12310,20 @@ define <vscale x 14 x bfloat> @vector_interleave_nxv14bf16_nxv2bf16(<vscale x 2
 ; ZVBB-NEXT:    add a7, a6, a2
 ; ZVBB-NEXT:    vle16.v v8, (a7)
 ; ZVBB-NEXT:    vle16.v v10, (a6)
-; ZVBB-NEXT:    add a6, a1, a1
+; ZVBB-NEXT:    srli a1, a1, 2
 ; ZVBB-NEXT:    add a2, a7, a2
 ; ZVBB-NEXT:    vle16.v v12, (a5)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a5, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
 ; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v11, (a2)
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v12, a1
 ; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v12, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a6, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
@@ -12889,7 +12824,6 @@ define <vscale x 7 x float> @vector_interleave_nxv7f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    addi a0, sp, 16
 ; CHECK-NEXT:    csrr a1, vlenb
 ; CHECK-NEXT:    srli a2, a1, 1
-; CHECK-NEXT:    srli a1, a1, 3
 ; CHECK-NEXT:    add a3, a0, a2
 ; CHECK-NEXT:    add a4, a3, a2
 ; CHECK-NEXT:    add a5, a4, a2
@@ -12899,20 +12833,20 @@ define <vscale x 7 x float> @vector_interleave_nxv7f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    add a7, a6, a2
 ; CHECK-NEXT:    vle32.v v8, (a7)
 ; CHECK-NEXT:    vle32.v v10, (a6)
-; CHECK-NEXT:    add a6, a1, a1
+; CHECK-NEXT:    srli a1, a1, 3
 ; CHECK-NEXT:    add a2, a7, a2
 ; CHECK-NEXT:    vle32.v v12, (a5)
-; CHECK-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v8, a1
 ; CHECK-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v11, (a2)
 ; CHECK-NEXT:    vle32.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v12, a1
 ; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v12, (a3)
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -12929,7 +12863,6 @@ define <vscale x 7 x float> @vector_interleave_nxv7f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    addi a0, sp, 16
 ; ZVBB-NEXT:    csrr a1, vlenb
 ; ZVBB-NEXT:    srli a2, a1, 1
-; ZVBB-NEXT:    srli a1, a1, 3
 ; ZVBB-NEXT:    add a3, a0, a2
 ; ZVBB-NEXT:    add a4, a3, a2
 ; ZVBB-NEXT:    add a5, a4, a2
@@ -12939,20 +12872,20 @@ define <vscale x 7 x float> @vector_interleave_nxv7f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    add a7, a6, a2
 ; ZVBB-NEXT:    vle32.v v8, (a7)
 ; ZVBB-NEXT:    vle32.v v10, (a6)
-; ZVBB-NEXT:    add a6, a1, a1
+; ZVBB-NEXT:    srli a1, a1, 3
 ; ZVBB-NEXT:    add a2, a7, a2
 ; ZVBB-NEXT:    vle32.v v12, (a5)
-; ZVBB-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a5, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v8, a1
 ; ZVBB-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v11, (a2)
 ; ZVBB-NEXT:    vle32.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v12, a1
 ; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v12, (a3)
 ; ZVBB-NEXT:    vle32.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a6, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
@@ -13945,23 +13878,22 @@ define <vscale x 16 x half> @vector_interleave_nxv16f16_nxv2f16(<vscale x 2 x ha
 ; CHECK-NEXT:    vle16.v v11, (t0)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle16.v v9, (a7)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v11, v8, a1
-; CHECK-NEXT:    vsetvli a7, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v10, (a6)
 ; CHECK-NEXT:    vle16.v v8, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v9, a1
-; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v12, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -13990,23 +13922,22 @@ define <vscale x 16 x half> @vector_interleave_nxv16f16_nxv2f16(<vscale x 2 x ha
 ; ZVBB-NEXT:    vle16.v v11, (t0)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle16.v v9, (a7)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v11, v8, a1
-; ZVBB-NEXT:    vsetvli a7, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v10, (a6)
 ; ZVBB-NEXT:    vle16.v v8, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v9, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v12, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
@@ -14243,23 +14174,22 @@ define <vscale x 16 x bfloat> @vector_interleave_nxv16bf16_nxv2bf16(<vscale x 2
 ; CHECK-NEXT:    vle16.v v11, (t0)
 ; CHECK-NEXT:    vle16.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 2
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle16.v v9, (a7)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v11, v8, a1
-; CHECK-NEXT:    vsetvli a7, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v10, (a6)
 ; CHECK-NEXT:    vle16.v v8, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v9, a1
-; CHECK-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; CHECK-NEXT:    vle16.v v12, (a3)
 ; CHECK-NEXT:    vle16.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -14288,23 +14218,22 @@ define <vscale x 16 x bfloat> @vector_interleave_nxv16bf16_nxv2bf16(<vscale x 2
 ; ZVBB-NEXT:    vle16.v v11, (t0)
 ; ZVBB-NEXT:    vle16.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 2
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle16.v v9, (a7)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v11, v8, a1
-; ZVBB-NEXT:    vsetvli a7, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v10, (a6)
 ; ZVBB-NEXT:    vle16.v v8, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v9, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e16, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e16, mf2, ta, ma
 ; ZVBB-NEXT:    vle16.v v12, (a3)
 ; ZVBB-NEXT:    vle16.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e16, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e16, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
@@ -14541,23 +14470,22 @@ define <vscale x 8 x float> @vector_interleave_nxv8f32_nxv1f32(<vscale x 1 x flo
 ; CHECK-NEXT:    vle32.v v11, (t0)
 ; CHECK-NEXT:    vle32.v v8, (a2)
 ; CHECK-NEXT:    srli a1, a1, 3
-; CHECK-NEXT:    add a2, a1, a1
 ; CHECK-NEXT:    vle32.v v9, (a7)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v11, v8, a1
-; CHECK-NEXT:    vsetvli a7, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v10, (a6)
 ; CHECK-NEXT:    vle32.v v8, (a5)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v10, v9, a1
-; CHECK-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v9, (a4)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v9, v8, a1
-; CHECK-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; CHECK-NEXT:    vle32.v v12, (a3)
 ; CHECK-NEXT:    vle32.v v8, (a0)
-; CHECK-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; CHECK-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; CHECK-NEXT:    vslideup.vx v8, v12, a1
 ; CHECK-NEXT:    csrr a0, vlenb
 ; CHECK-NEXT:    slli a0, a0, 2
@@ -14586,23 +14514,22 @@ define <vscale x 8 x float> @vector_interleave_nxv8f32_nxv1f32(<vscale x 1 x flo
 ; ZVBB-NEXT:    vle32.v v11, (t0)
 ; ZVBB-NEXT:    vle32.v v8, (a2)
 ; ZVBB-NEXT:    srli a1, a1, 3
-; ZVBB-NEXT:    add a2, a1, a1
 ; ZVBB-NEXT:    vle32.v v9, (a7)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v11, v8, a1
-; ZVBB-NEXT:    vsetvli a7, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v10, (a6)
 ; ZVBB-NEXT:    vle32.v v8, (a5)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v10, v9, a1
-; ZVBB-NEXT:    vsetvli a5, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v9, (a4)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v9, v8, a1
-; ZVBB-NEXT:    vsetvli a4, zero, e32, mf2, ta, ma
+; ZVBB-NEXT:    vsetvli a2, zero, e32, mf2, ta, ma
 ; ZVBB-NEXT:    vle32.v v12, (a3)
 ; ZVBB-NEXT:    vle32.v v8, (a0)
-; ZVBB-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; ZVBB-NEXT:    vsetvli a0, zero, e32, m1, ta, ma
 ; ZVBB-NEXT:    vslideup.vx v8, v12, a1
 ; ZVBB-NEXT:    csrr a0, vlenb
 ; ZVBB-NEXT:    slli a0, a0, 2
diff --git a/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll b/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
index df7af4d8b166..111fa368ac15 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vfptoi-sdnode.ll
@@ -634,12 +634,11 @@ define <vscale x 32 x i1> @vfptosi_nxv32bf16_nxv32i1(<vscale x 32 x bfloat> %va)
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vfncvt.rtz.x.f.w v8, v16
 ; CHECK-NEXT:    vfncvt.rtz.x.f.w v12, v24
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vand.vi v12, v12, 1
 ; CHECK-NEXT:    vmsne.vi v16, v8, 0
 ; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v0, v16, a0
 ; CHECK-NEXT:    ret
   %evec = fptosi <vscale x 32 x bfloat> %va to <vscale x 32 x i1>
@@ -656,12 +655,11 @@ define <vscale x 32 x i1> @vfptoui_nxv32bf16_nxv32i1(<vscale x 32 x bfloat> %va)
 ; CHECK-NEXT:    srli a0, a0, 2
 ; CHECK-NEXT:    vfncvt.rtz.xu.f.w v8, v16
 ; CHECK-NEXT:    vfncvt.rtz.xu.f.w v12, v24
-; CHECK-NEXT:    add a1, a0, a0
 ; CHECK-NEXT:    vand.vi v8, v8, 1
 ; CHECK-NEXT:    vand.vi v12, v12, 1
 ; CHECK-NEXT:    vmsne.vi v16, v8, 0
 ; CHECK-NEXT:    vmsne.vi v0, v12, 0
-; CHECK-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; CHECK-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vx v0, v16, a0
 ; CHECK-NEXT:    ret
   %evec = fptoui <vscale x 32 x bfloat> %va to <vscale x 32 x i1>
@@ -1654,12 +1652,11 @@ define <vscale x 32 x i1> @vfptosi_nxv32f16_nxv32i1(<vscale x 32 x half> %va) {
 ; ZVFHMIN-NEXT:    srli a0, a0, 2
 ; ZVFHMIN-NEXT:    vfncvt.rtz.x.f.w v8, v16
 ; ZVFHMIN-NEXT:    vfncvt.rtz.x.f.w v12, v24
-; ZVFHMIN-NEXT:    add a1, a0, a0
 ; ZVFHMIN-NEXT:    vand.vi v8, v8, 1
 ; ZVFHMIN-NEXT:    vand.vi v12, v12, 1
 ; ZVFHMIN-NEXT:    vmsne.vi v16, v8, 0
 ; ZVFHMIN-NEXT:    vmsne.vi v0, v12, 0
-; ZVFHMIN-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslideup.vx v0, v16, a0
 ; ZVFHMIN-NEXT:    ret
   %evec = fptosi <vscale x 32 x half> %va to <vscale x 32 x i1>
@@ -1684,12 +1681,11 @@ define <vscale x 32 x i1> @vfptoui_nxv32f16_nxv32i1(<vscale x 32 x half> %va) {
 ; ZVFHMIN-NEXT:    srli a0, a0, 2
 ; ZVFHMIN-NEXT:    vfncvt.rtz.xu.f.w v8, v16
 ; ZVFHMIN-NEXT:    vfncvt.rtz.xu.f.w v12, v24
-; ZVFHMIN-NEXT:    add a1, a0, a0
 ; ZVFHMIN-NEXT:    vand.vi v8, v8, 1
 ; ZVFHMIN-NEXT:    vand.vi v12, v12, 1
 ; ZVFHMIN-NEXT:    vmsne.vi v16, v8, 0
 ; ZVFHMIN-NEXT:    vmsne.vi v0, v12, 0
-; ZVFHMIN-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
+; ZVFHMIN-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
 ; ZVFHMIN-NEXT:    vslideup.vx v0, v16, a0
 ; ZVFHMIN-NEXT:    ret
   %evec = fptoui <vscale x 32 x half> %va to <vscale x 32 x i1>
diff --git a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
index 142ee5256f9e..186815405227 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vp-vector-interleaved-access.ll
@@ -567,38 +567,37 @@ define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %
 ; RV32:       # %bb.0:
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v8, v0
-; RV32-NEXT:    slli a2, a1, 1
 ; RV32-NEXT:    vmv.v.i v9, 0
-; RV32-NEXT:    li a1, -1
+; RV32-NEXT:    li a2, -1
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; RV32-NEXT:    vmv.v.i v10, 0
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmerge.vim v11, v9, 1, v0
-; RV32-NEXT:    srli a3, a3, 2
 ; RV32-NEXT:    vwaddu.vv v12, v11, v11
-; RV32-NEXT:    vwmaccu.vx v12, a1, v11
+; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    srli a2, a2, 2
 ; RV32-NEXT:    vmsne.vi v0, v12, 0
-; RV32-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v11, v12, a3
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v11, v12, a2
 ; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
-; RV32-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v11, 0
-; RV32-NEXT:    add a1, a3, a3
+; RV32-NEXT:    slli a3, a1, 1
 ; RV32-NEXT:    vmerge.vim v9, v9, 1, v0
-; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v9, a3
-; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
+; RV32-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v9, a2
+; RV32-NEXT:    vsetvli zero, a3, e8, mf2, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
 ; RV32-NEXT:    li a1, 32
-; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli a2, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vnsrl.wx v13, v10, a1
 ; RV32-NEXT:    vmv.x.s a1, v10
 ; RV32-NEXT:    vnsrl.wi v12, v10, 0
-; RV32-NEXT:    srli a2, a2, 1
+; RV32-NEXT:    srli a3, a3, 1
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    vsetvli zero, a2, e32, m1, ta, ma
+; RV32-NEXT:    vsetvli zero, a3, e32, m1, ta, ma
 ; RV32-NEXT:    vsseg2e32.v v12, (a0), v0.t
 ; RV32-NEXT:    mv a0, a1
 ; RV32-NEXT:    ret
@@ -611,26 +610,24 @@ define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %
 ; RV64-NEXT:    li a2, -1
 ; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; RV64-NEXT:    vmv.v.i v10, 0
-; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a4, a1, 33
-; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmerge.vim v11, v9, 1, v0
-; RV64-NEXT:    srli a3, a3, 2
 ; RV64-NEXT:    vwaddu.vv v12, v11, v11
 ; RV64-NEXT:    vwmaccu.vx v12, a2, v11
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    srli a2, a2, 2
 ; RV64-NEXT:    vmsne.vi v0, v12, 0
-; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
-; RV64-NEXT:    vslidedown.vx v11, v12, a3
+; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vx v11, v12, a2
 ; RV64-NEXT:    vmerge.vim v10, v10, 1, v0
-; RV64-NEXT:    vsetvli a1, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v11, 0
-; RV64-NEXT:    add a1, a3, a3
+; RV64-NEXT:    slli a3, a1, 33
 ; RV64-NEXT:    vmerge.vim v9, v9, 1, v0
-; RV64-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
-; RV64-NEXT:    vslideup.vx v10, v9, a3
 ; RV64-NEXT:    vsetvli a1, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslideup.vx v10, v9, a2
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
-; RV64-NEXT:    srli a1, a4, 32
+; RV64-NEXT:    srli a1, a3, 32
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
 ; RV64-NEXT:    vle32.v v10, (a0), v0.t
 ; RV64-NEXT:    li a1, 32
@@ -638,9 +635,9 @@ define i32 @masked_load_store_factor2_v2_shared_mask_extract(<vscale x 2 x i1> %
 ; RV64-NEXT:    vnsrl.wx v13, v10, a1
 ; RV64-NEXT:    vmv.x.s a1, v10
 ; RV64-NEXT:    vnsrl.wi v12, v10, 0
-; RV64-NEXT:    srli a4, a4, 33
+; RV64-NEXT:    srli a3, a3, 33
 ; RV64-NEXT:    vmv1r.v v0, v8
-; RV64-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
+; RV64-NEXT:    vsetvli zero, a3, e32, m1, ta, ma
 ; RV64-NEXT:    vsseg2e32.v v12, (a0), v0.t
 ; RV64-NEXT:    mv a0, a1
 ; RV64-NEXT:    ret
@@ -807,10 +804,7 @@ define void @not_balanced_store_tree(<vscale x 1 x i32> %v0, <vscale x 2 x i32>
 ; RV32-NEXT:    srli a3, a3, 3
 ; RV32-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vslidedown.vx v8, v12, a3
-; RV32-NEXT:    add a4, a3, a3
-; RV32-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
 ; RV32-NEXT:    vslideup.vx v12, v8, a3
-; RV32-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; RV32-NEXT:    vwaddu.vv v16, v12, v9
 ; RV32-NEXT:    vwmaccu.vx v16, a2, v9
 ; RV32-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
@@ -831,10 +825,7 @@ define void @not_balanced_store_tree(<vscale x 1 x i32> %v0, <vscale x 2 x i32>
 ; RV64-NEXT:    srli a3, a3, 3
 ; RV64-NEXT:    vsetvli a4, zero, e32, m1, ta, ma
 ; RV64-NEXT:    vslidedown.vx v8, v12, a3
-; RV64-NEXT:    add a4, a3, a3
-; RV64-NEXT:    vsetvli zero, a4, e32, m1, ta, ma
 ; RV64-NEXT:    vslideup.vx v12, v8, a3
-; RV64-NEXT:    vsetvli a3, zero, e32, m1, ta, ma
 ; RV64-NEXT:    vwaddu.vv v16, v12, v9
 ; RV64-NEXT:    vwmaccu.vx v16, a2, v9
 ; RV64-NEXT:    vsetvli a3, zero, e32, m2, ta, ma
@@ -858,29 +849,28 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1>
 ; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmv1r.v v9, v0
 ; RV32-NEXT:    vmv1r.v v0, v8
-; RV32-NEXT:    slli a1, a1, 1
 ; RV32-NEXT:    vmv.v.i v8, 0
 ; RV32-NEXT:    li a2, -1
 ; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; RV32-NEXT:    vmv.v.i v10, 0
-; RV32-NEXT:    csrr a3, vlenb
-; RV32-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmerge.vim v11, v8, 1, v0
 ; RV32-NEXT:    vmv1r.v v0, v9
 ; RV32-NEXT:    vmerge.vim v9, v8, 1, v0
-; RV32-NEXT:    srli a3, a3, 2
 ; RV32-NEXT:    vwaddu.vv v12, v9, v11
 ; RV32-NEXT:    vwmaccu.vx v12, a2, v11
+; RV32-NEXT:    csrr a2, vlenb
+; RV32-NEXT:    srli a2, a2, 2
 ; RV32-NEXT:    vmsne.vi v0, v12, 0
-; RV32-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
-; RV32-NEXT:    vslidedown.vx v9, v12, a3
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslidedown.vx v9, v12, a2
 ; RV32-NEXT:    vmerge.vim v10, v10, 1, v0
-; RV32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV32-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v9, 0
-; RV32-NEXT:    add a2, a3, a3
+; RV32-NEXT:    slli a1, a1, 1
 ; RV32-NEXT:    vmerge.vim v8, v8, 1, v0
-; RV32-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vx v10, v8, a3
+; RV32-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV32-NEXT:    vslideup.vx v10, v8, a2
 ; RV32-NEXT:    vsetvli zero, a1, e8, mf2, ta, ma
 ; RV32-NEXT:    vmsne.vi v0, v10, 0
 ; RV32-NEXT:    vle32.v v10, (a0), v0.t
@@ -899,26 +889,24 @@ define {<vscale x 2 x i32>, <vscale x 2 x i32>} @not_same_mask(<vscale x 2 x i1>
 ; RV64-NEXT:    li a2, -1
 ; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
 ; RV64-NEXT:    vmv.v.i v10, 0
-; RV64-NEXT:    csrr a3, vlenb
-; RV64-NEXT:    slli a1, a1, 33
-; RV64-NEXT:    vsetvli a4, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmerge.vim v11, v8, 1, v0
 ; RV64-NEXT:    vmv1r.v v0, v9
 ; RV64-NEXT:    vmerge.vim v9, v8, 1, v0
-; RV64-NEXT:    srli a3, a3, 2
 ; RV64-NEXT:    vwaddu.vv v12, v9, v11
 ; RV64-NEXT:    vwmaccu.vx v12, a2, v11
+; RV64-NEXT:    csrr a2, vlenb
+; RV64-NEXT:    srli a2, a2, 2
 ; RV64-NEXT:    vmsne.vi v0, v12, 0
-; RV64-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
-; RV64-NEXT:    vslidedown.vx v9, v12, a3
+; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslidedown.vx v9, v12, a2
 ; RV64-NEXT:    vmerge.vim v10, v10, 1, v0
-; RV64-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
+; RV64-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
 ; RV64-NEXT:    vmsne.vi v0, v9, 0
-; RV64-NEXT:    add a2, a3, a3
+; RV64-NEXT:    slli a1, a1, 33
 ; RV64-NEXT:    vmerge.vim v8, v8, 1, v0
-; RV64-NEXT:    vsetvli zero, a2, e8, mf2, ta, ma
-; RV64-NEXT:    vslideup.vx v10, v8, a3
-; RV64-NEXT:    vsetvli a2, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; RV64-NEXT:    vslideup.vx v10, v8, a2
 ; RV64-NEXT:    vmsne.vi v0, v10, 0
 ; RV64-NEXT:    srli a1, a1, 32
 ; RV64-NEXT:    vsetvli zero, a1, e32, m2, ta, ma
-- 
cgit v1.2.3


From 74687180dde07312521db09c6f6454fe9d1e5662 Mon Sep 17 00:00:00 2001
From: Kirill Chibisov <contact@kchibisov.com>
Date: Wed, 18 Jun 2025 14:38:47 +0900
Subject: [mlir][emitc] Make CExpression trait into interface (#142771)

By defining `CExpressionInterface`, we move the side effect detection
logic from `emitc.expression` into the individual operations
implementing the interface allowing operations to gradually tune the
side effect.

It also allows checking for side effects each operation individually.
---
 mlir/include/mlir/Dialect/EmitC/IR/CMakeLists.txt  |   6 ++
 mlir/include/mlir/Dialect/EmitC/IR/EmitC.h         |   2 +-
 mlir/include/mlir/Dialect/EmitC/IR/EmitC.td        | 107 ++++++++++++---------
 .../mlir/Dialect/EmitC/IR/EmitCInterfaces.h        |  31 ++++++
 .../mlir/Dialect/EmitC/IR/EmitCInterfaces.td       |  48 +++++++++
 mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h   |  30 ------
 mlir/lib/Dialect/EmitC/IR/EmitC.cpp                |   6 +-
 .../Dialect/EmitC/Transforms/FormExpressions.cpp   |   2 +-
 mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp   |   3 +-
 mlir/lib/Target/Cpp/TranslateToCpp.cpp             |   6 +-
 10 files changed, 159 insertions(+), 82 deletions(-)
 create mode 100644 mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.h
 create mode 100644 mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td
 delete mode 100644 mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h

diff --git a/mlir/include/mlir/Dialect/EmitC/IR/CMakeLists.txt b/mlir/include/mlir/Dialect/EmitC/IR/CMakeLists.txt
index 610170f5944e..299cee76cb1b 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/EmitC/IR/CMakeLists.txt
@@ -1,6 +1,12 @@
 add_mlir_dialect(EmitC emitc)
 add_mlir_doc(EmitC EmitC Dialects/ -gen-dialect-doc -dialect emitc)
 
+set(LLVM_TARGET_DEFINITIONS EmitCInterfaces.td)
+mlir_tablegen(EmitCInterfaces.h.inc -gen-op-interface-decls)
+mlir_tablegen(EmitCInterfaces.cpp.inc -gen-op-interface-defs)
+add_public_tablegen_target(MLIREmitCInterfacesIncGen)
+add_dependencies(mlir-generic-headers MLIREmitCInterfacesIncGen)
+
 set(LLVM_TARGET_DEFINITIONS EmitCAttributes.td)
 mlir_tablegen(EmitCEnums.h.inc -gen-enum-decls)
 mlir_tablegen(EmitCEnums.cpp.inc -gen-enum-defs)
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h
index 57029c64ffd0..1984ed8a7f06 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h
@@ -14,7 +14,7 @@
 #define MLIR_DIALECT_EMITC_IR_EMITC_H
 
 #include "mlir/Bytecode/BytecodeOpInterface.h"
-#include "mlir/Dialect/EmitC/IR/EmitCTraits.h"
+#include "mlir/Dialect/EmitC/IR/EmitCInterfaces.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinOps.h"
 #include "mlir/IR/BuiltinTypes.h"
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
index e53d3e45875d..9ecdb74f4d82 100644
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td
@@ -14,6 +14,7 @@
 #define MLIR_DIALECT_EMITC_IR_EMITC
 
 include "mlir/Dialect/EmitC/IR/EmitCAttributes.td"
+include "mlir/Dialect/EmitC/IR/EmitCInterfaces.td"
 include "mlir/Dialect/EmitC/IR/EmitCTypes.td"
 
 include "mlir/Interfaces/CallInterfaces.td"
@@ -35,22 +36,31 @@ class EmitC_Op<string mnemonic, list<Trait> traits = []>
 
 // Base class for unary operations.
 class EmitC_UnaryOp<string mnemonic, list<Trait> traits = []> :
-    EmitC_Op<mnemonic, traits> {
+    EmitC_Op<mnemonic, !listconcat(traits, [CExpressionInterface])> {
   let arguments = (ins EmitCType);
   let results = (outs EmitCType);
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
+
+  let extraClassDeclaration = [{
+    bool hasSideEffects() {
+      return false;
+    }
+  }];
 }
 
 // Base class for binary operations.
 class EmitC_BinaryOp<string mnemonic, list<Trait> traits = []> :
-    EmitC_Op<mnemonic, traits> {
+    EmitC_Op<mnemonic, !listconcat(traits, [CExpressionInterface])> {
   let arguments = (ins EmitCType:$lhs, EmitCType:$rhs);
   let results = (outs EmitCType);
   let assemblyFormat = "operands attr-dict `:` functional-type(operands, results)";
-}
 
-// EmitC OpTrait
-def CExpression : NativeOpTrait<"emitc::CExpression">;
+  let extraClassDeclaration = [{
+    bool hasSideEffects() {
+      return false;
+    }
+  }];
+}
 
 // Types only used in binary arithmetic operations.
 def IntegerIndexOrOpaqueType : Type<CPred<"emitc::isIntegerIndexOrOpaqueType($_self)">,
@@ -103,7 +113,7 @@ def EmitC_FileOp
   let skipDefaultBuilders = 1;
 }
 
-def EmitC_AddOp : EmitC_BinaryOp<"add", [CExpression]> {
+def EmitC_AddOp : EmitC_BinaryOp<"add", []> {
   let summary = "Addition operation";
   let description = [{
     With the `emitc.add` operation the arithmetic operator + (addition) can
@@ -126,7 +136,7 @@ def EmitC_AddOp : EmitC_BinaryOp<"add", [CExpression]> {
   let hasVerifier = 1;
 }
 
-def EmitC_ApplyOp : EmitC_Op<"apply", [CExpression]> {
+def EmitC_ApplyOp : EmitC_Op<"apply", [CExpressionInterface]> {
   let summary = "Apply operation";
   let description = [{
     With the `emitc.apply` operation the operators & (address of) and * (contents of)
@@ -152,10 +162,17 @@ def EmitC_ApplyOp : EmitC_Op<"apply", [CExpression]> {
   let assemblyFormat = [{
     $applicableOperator `(` $operand `)` attr-dict `:` functional-type($operand, results)
   }];
+
+  let extraClassDeclaration = [{
+    bool hasSideEffects() {
+      return getApplicableOperator() == "*";
+    }
+  }];
+
   let hasVerifier = 1;
 }
 
-def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", [CExpression]> {
+def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", []> {
   let summary = "Bitwise and operation";
   let description = [{
     With the `emitc.bitwise_and` operation the bitwise operator & (and) can
@@ -173,8 +190,7 @@ def EmitC_BitwiseAndOp : EmitC_BinaryOp<"bitwise_and", [CExpression]> {
   }];
 }
 
-def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift",
-    [CExpression]> {
+def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift", []> {
   let summary = "Bitwise left shift operation";
   let description = [{
     With the `emitc.bitwise_left_shift` operation the bitwise operator <<
@@ -192,7 +208,7 @@ def EmitC_BitwiseLeftShiftOp : EmitC_BinaryOp<"bitwise_left_shift",
   }];
 }
 
-def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", [CExpression]> {
+def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", []> {
   let summary = "Bitwise not operation";
   let description = [{
     With the `emitc.bitwise_not` operation the bitwise operator ~ (not) can
@@ -210,7 +226,7 @@ def EmitC_BitwiseNotOp : EmitC_UnaryOp<"bitwise_not", [CExpression]> {
   }];
 }
 
-def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", [CExpression]> {
+def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", []> {
   let summary = "Bitwise or operation";
   let description = [{
     With the `emitc.bitwise_or` operation the bitwise operator | (or)
@@ -228,8 +244,7 @@ def EmitC_BitwiseOrOp : EmitC_BinaryOp<"bitwise_or", [CExpression]> {
   }];
 }
 
-def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift",
-    [CExpression]> {
+def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift", []> {
   let summary = "Bitwise right shift operation";
   let description = [{
     With the `emitc.bitwise_right_shift` operation the bitwise operator >>
@@ -247,7 +262,7 @@ def EmitC_BitwiseRightShiftOp : EmitC_BinaryOp<"bitwise_right_shift",
   }];
 }
 
-def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", [CExpression]> {
+def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", []> {
   let summary = "Bitwise xor operation";
   let description = [{
     With the `emitc.bitwise_xor` operation the bitwise operator ^ (xor)
@@ -265,7 +280,7 @@ def EmitC_BitwiseXorOp : EmitC_BinaryOp<"bitwise_xor", [CExpression]> {
   }];
 }
 
-def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", [CExpression]> {
+def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", [CExpressionInterface]> {
   let summary = "Opaque call operation";
   let description = [{
     The `emitc.call_opaque` operation represents a C++ function call. The callee
@@ -312,7 +327,7 @@ def EmitC_CallOpaqueOp : EmitC_Op<"call_opaque", [CExpression]> {
 }
 
 def EmitC_CastOp : EmitC_Op<"cast",
-    [CExpression,
+    [CExpressionInterface,
      DeclareOpInterfaceMethods<CastOpInterface>]> {
   let summary = "Cast operation";
   let description = [{
@@ -335,9 +350,15 @@ def EmitC_CastOp : EmitC_Op<"cast",
   let arguments = (ins EmitCType:$source);
   let results = (outs EmitCType:$dest);
   let assemblyFormat = "$source attr-dict `:` type($source) `to` type($dest)";
+
+  let extraClassDeclaration = [{
+    bool hasSideEffects() {
+      return false;
+    }
+  }];
 }
 
-def EmitC_CmpOp : EmitC_BinaryOp<"cmp", [CExpression]> {
+def EmitC_CmpOp : EmitC_BinaryOp<"cmp", []> {
   let summary = "Comparison operation";
   let description = [{
     With the `emitc.cmp` operation the comparison operators ==, !=, <, <=, >, >=, <=> 
@@ -407,7 +428,7 @@ def EmitC_ConstantOp : EmitC_Op<"constant", [ConstantLike]> {
   let hasVerifier = 1;
 }
 
-def EmitC_DivOp : EmitC_BinaryOp<"div", [CExpression]> {
+def EmitC_DivOp : EmitC_BinaryOp<"div", []> {
   let summary = "Division operation";
   let description = [{
     With the `emitc.div` operation the arithmetic operator / (division) can
@@ -462,7 +483,7 @@ def EmitC_ExpressionOp : EmitC_Op<"expression",
     ```
 
     The operations allowed within expression body are EmitC operations with the
-    CExpression trait.
+    CExpressionInterface interface.
 
     When specified, the optional `do_not_inline` indicates that the expression is
     to be emitted as seen above, i.e. as the rhs of an EmitC SSA value
@@ -480,18 +501,8 @@ def EmitC_ExpressionOp : EmitC_Op<"expression",
   let extraClassDeclaration = [{
     bool hasSideEffects() {
       auto predicate = [](Operation &op) {
-        assert(op.hasTrait<OpTrait::emitc::CExpression>() && "Expected a C expression");
-        // Conservatively assume calls to read and write memory.
-        if (isa<emitc::CallOpaqueOp>(op))
-          return true;
-        // De-referencing reads modifiable memory, address-taking has no
-        // side-effect.
-        auto applyOp = dyn_cast<emitc::ApplyOp>(op);
-        if (applyOp)
-          return applyOp.getApplicableOperator() == "*";
-        // Any load operation is assumed to read from memory and thus perform
-        // a side effect.
-        return isa<emitc::LoadOp>(op);
+        assert(isa<emitc::CExpressionInterface>(op) && "Expected a C expression");
+        return cast<emitc::CExpressionInterface>(op).hasSideEffects();
       };
       return llvm::any_of(getRegion().front().without_terminator(), predicate);
     };
@@ -579,7 +590,7 @@ def EmitC_ForOp : EmitC_Op<"for",
 }
 
 def EmitC_CallOp : EmitC_Op<"call",
-    [CallOpInterface, CExpression,
+    [CallOpInterface, CExpressionInterface,
      DeclareOpInterfaceMethods<SymbolUserOpInterface>]> {
   let summary = "Call operation";
   let description = [{
@@ -649,6 +660,10 @@ def EmitC_CallOp : EmitC_Op<"call",
     void setCalleeFromCallable(CallInterfaceCallable callee) {
       (*this)->setAttr("callee", cast<SymbolRefAttr>(callee));
     }
+
+    bool hasSideEffects() {
+      return false;
+    }
   }];
 
   let assemblyFormat = [{
@@ -861,7 +876,7 @@ def EmitC_LiteralOp : EmitC_Op<"literal", [Pure]> {
   let assemblyFormat = "$value attr-dict `:` type($result)";
 }
 
-def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", [CExpression]> {
+def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", []> {
   let summary = "Logical and operation";
   let description = [{
     With the `emitc.logical_and` operation the logical operator && (and) can
@@ -882,7 +897,7 @@ def EmitC_LogicalAndOp : EmitC_BinaryOp<"logical_and", [CExpression]> {
   let assemblyFormat = "operands attr-dict `:` type(operands)";
 }
 
-def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", [CExpression]> {
+def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", []> {
   let summary = "Logical not operation";
   let description = [{
     With the `emitc.logical_not` operation the logical operator ! (negation) can
@@ -903,7 +918,7 @@ def EmitC_LogicalNotOp : EmitC_UnaryOp<"logical_not", [CExpression]> {
   let assemblyFormat = "operands attr-dict `:` type(operands)";
 }
 
-def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", [CExpression]> {
+def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", []> {
   let summary = "Logical or operation";
   let description = [{
     With the `emitc.logical_or` operation the logical operator || (inclusive or)
@@ -924,7 +939,7 @@ def EmitC_LogicalOrOp : EmitC_BinaryOp<"logical_or", [CExpression]> {
   let assemblyFormat = "operands attr-dict `:` type(operands)";
 }
 
-def EmitC_LoadOp : EmitC_Op<"load", [CExpression,
+def EmitC_LoadOp : EmitC_Op<"load", [CExpressionInterface,
   TypesMatchWith<"result type matches value type of 'operand'",
                   "operand", "result",
                   "::llvm::cast<LValueType>($_self).getValueType()">
@@ -953,7 +968,7 @@ def EmitC_LoadOp : EmitC_Op<"load", [CExpression,
   let assemblyFormat = "$operand attr-dict `:` type($operand)"; 
 }
 
-def EmitC_MulOp : EmitC_BinaryOp<"mul", [CExpression]> {
+def EmitC_MulOp : EmitC_BinaryOp<"mul", []> {
   let summary = "Multiplication operation";
   let description = [{
     With the `emitc.mul` operation the arithmetic operator * (multiplication) can
@@ -977,7 +992,7 @@ def EmitC_MulOp : EmitC_BinaryOp<"mul", [CExpression]> {
   let results = (outs FloatIntegerIndexOrOpaqueType);
 }
 
-def EmitC_RemOp : EmitC_BinaryOp<"rem", [CExpression]> {
+def EmitC_RemOp : EmitC_BinaryOp<"rem", []> {
   let summary = "Remainder operation";
   let description = [{
     With the `emitc.rem` operation the arithmetic operator % (remainder) can
@@ -999,7 +1014,7 @@ def EmitC_RemOp : EmitC_BinaryOp<"rem", [CExpression]> {
   let results = (outs IntegerIndexOrOpaqueType);
 }
 
-def EmitC_SubOp : EmitC_BinaryOp<"sub", [CExpression]> {
+def EmitC_SubOp : EmitC_BinaryOp<"sub", []> {
   let summary = "Subtraction operation";
   let description = [{
     With the `emitc.sub` operation the arithmetic operator - (subtraction) can
@@ -1069,7 +1084,7 @@ def EmitC_MemberOfPtrOp : EmitC_Op<"member_of_ptr"> {
 }
 
 def EmitC_ConditionalOp : EmitC_Op<"conditional",
-    [AllTypesMatch<["true_value", "false_value", "result"]>, CExpression]> {
+    [AllTypesMatch<["true_value", "false_value", "result"]>, CExpressionInterface]> {
   let summary = "Conditional (ternary) operation";
   let description = [{
     With the `emitc.conditional` operation the ternary conditional operator can
@@ -1096,9 +1111,15 @@ def EmitC_ConditionalOp : EmitC_Op<"conditional",
   let arguments = (ins I1:$condition, EmitCType:$true_value, EmitCType:$false_value);
   let results = (outs EmitCType:$result);
   let assemblyFormat = "operands attr-dict `:` type($result)";
+
+  let extraClassDeclaration = [{
+    bool hasSideEffects() {
+      return false;
+    }
+  }];
 }
 
-def EmitC_UnaryMinusOp : EmitC_UnaryOp<"unary_minus", [CExpression]> {
+def EmitC_UnaryMinusOp : EmitC_UnaryOp<"unary_minus", []> {
   let summary = "Unary minus operation";
   let description = [{
     With the `emitc.unary_minus` operation the unary operator - (minus) can be
@@ -1116,7 +1137,7 @@ def EmitC_UnaryMinusOp : EmitC_UnaryOp<"unary_minus", [CExpression]> {
   }];
 }
 
-def EmitC_UnaryPlusOp : EmitC_UnaryOp<"unary_plus", [CExpression]> {
+def EmitC_UnaryPlusOp : EmitC_UnaryOp<"unary_plus", []> {
   let summary = "Unary plus operation";
   let description = [{
     With the `emitc.unary_plus` operation the unary operator + (plus) can be
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.h
new file mode 100644
index 000000000000..51efe76aceb5
--- /dev/null
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.h
@@ -0,0 +1,31 @@
+//===- EmitCInterfaces.h - EmitC interfaces definitions ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares C++ classes for some of the interfaces used in the EmitC
+// dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_EMITC_IR_EMITCINTERFACES_H
+#define MLIR_DIALECT_EMITC_IR_EMITCINTERFACES_H
+
+#include "mlir/IR/OpDefinition.h"
+
+namespace mlir {
+namespace emitc {
+//
+} // namespace emitc
+} // namespace mlir
+
+//===----------------------------------------------------------------------===//
+// EmitC Dialect Interfaces
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/EmitC/IR/EmitCInterfaces.h.inc"
+
+#endif // MLIR_DIALECT_EMITC_IR_EMITCINTERFACES_H
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td
new file mode 100644
index 000000000000..777784e56202
--- /dev/null
+++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td
@@ -0,0 +1,48 @@
+//===- EmitCInterfaces.td - EmitC Interfaces ---------------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares the interfaces used by EmitC.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_EMITC_IR_EMITCINTERFACES
+#define MLIR_DIALECT_EMITC_IR_EMITCINTERFACES
+
+include "mlir/IR/OpBase.td"
+
+def CExpressionInterface : OpInterface<"CExpressionInterface"> {
+  let description = [{
+    Interface to mark operations that can be part of the CExpression.
+  }];
+
+  let cppNamespace = "::mlir::emitc";
+  let methods = [
+    InterfaceMethod<[{
+      Check whether operation has side effects that may affect the expression
+      evaluation.
+
+      By default operation is marked as having side effects.
+
+      ```c++
+      class ConcreteOp ... {
+      public:
+        bool hasSideEffects() {
+          // That way we can override the default implementation.
+          return false;
+        }
+      };
+      ```
+    }],
+      "bool", "hasSideEffects", (ins), /*methodBody=*/[{}],
+       /*defaultImplementation=*/[{
+        return true;
+    }]>,
+  ];
+}
+
+#endif // MLIR_DIALECT_EMITC_IR_EMITCINTERFACES
diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h
deleted file mode 100644
index c1602dfce4b4..000000000000
--- a/mlir/include/mlir/Dialect/EmitC/IR/EmitCTraits.h
+++ /dev/null
@@ -1,30 +0,0 @@
-//===- EmitCTraits.h - EmitC trait definitions ------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares C++ classes for some of the traits used in the EmitC
-// dialect.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H
-#define MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H
-
-#include "mlir/IR/OpDefinition.h"
-
-namespace mlir {
-namespace OpTrait {
-namespace emitc {
-
-template <typename ConcreteType>
-class CExpression : public TraitBase<ConcreteType, CExpression> {};
-
-} // namespace emitc
-} // namespace OpTrait
-} // namespace mlir
-
-#endif // MLIR_DIALECT_EMITC_IR_EMITCTRAITS_H
diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
index f82b20712b8c..e602210c2dc6 100644
--- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
+++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/EmitC/IR/EmitC.h"
-#include "mlir/Dialect/EmitC/IR/EmitCTraits.h"
+#include "mlir/Dialect/EmitC/IR/EmitCInterfaces.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
@@ -418,7 +418,7 @@ LogicalResult ExpressionOp::verify() {
     return emitOpError("requires yielded type to match return type");
 
   for (Operation &op : region.front().without_terminator()) {
-    if (!op.hasTrait<OpTrait::emitc::CExpression>())
+    if (!isa<emitc::CExpressionInterface>(op))
       return emitOpError("contains an unsupported operation");
     if (op.getNumResults() != 1)
       return emitOpError("requires exactly one result for each operation");
@@ -1404,5 +1404,7 @@ void FileOp::build(OpBuilder &builder, OperationState &state, StringRef id) {
 // TableGen'd op method definitions
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/EmitC/IR/EmitCInterfaces.cpp.inc"
+
 #define GET_OP_CLASSES
 #include "mlir/Dialect/EmitC/IR/EmitC.cpp.inc"
diff --git a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp
index 224d68ab8b4a..2f3e2618f4d7 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/FormExpressions.cpp
@@ -36,7 +36,7 @@ struct FormExpressionsPass
     // Wrap each C operator op with an expression op.
     OpBuilder builder(context);
     auto matchFun = [&](Operation *op) {
-      if (op->hasTrait<OpTrait::emitc::CExpression>() &&
+      if (isa<emitc::CExpressionInterface>(*op) &&
           !op->getParentOfType<emitc::ExpressionOp>() &&
           op->getNumResults() == 1)
         createExpression(op, builder);
diff --git a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
index 87350ecdceaa..a578a86b499a 100644
--- a/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
+++ b/mlir/lib/Dialect/EmitC/Transforms/Transforms.cpp
@@ -16,8 +16,7 @@ namespace mlir {
 namespace emitc {
 
 ExpressionOp createExpression(Operation *op, OpBuilder &builder) {
-  assert(op->hasTrait<OpTrait::emitc::CExpression>() &&
-         "Expected a C expression");
+  assert(isa<emitc::CExpressionInterface>(op) && "Expected a C expression");
 
   // Create an expression yielding the value returned by op.
   assert(op->getNumResults() == 1 && "Expected exactly one result");
diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
index 5abc112ab8c7..067a0470b14e 100644
--- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp
+++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp
@@ -329,9 +329,9 @@ static bool shouldBeInlined(ExpressionOp expressionOp) {
   if (hasDeferredEmission(user))
     return false;
 
-  // Do not inline expressions used by ops with the CExpression trait. If this
-  // was intended, the user could have been merged into the expression op.
-  return !user->hasTrait<OpTrait::emitc::CExpression>();
+  // Do not inline expressions used by ops with the CExpressionInterface. If
+  // this was intended, the user could have been merged into the expression op.
+  return !isa<emitc::CExpressionInterface>(*user);
 }
 
 static LogicalResult printConstantOp(CppEmitter &emitter, Operation *operation,
-- 
cgit v1.2.3


From 10f29a607205c0c17ee9249a66feb63f0fdae182 Mon Sep 17 00:00:00 2001
From: Kunqiu Chen <camsyn@foxmail.com>
Date: Wed, 18 Jun 2025 14:53:33 +0800
Subject: [MSan] Fix wrong unpoison size in SignalAction (#144071)

MSan should unpoison the parameters of extended signal handlers.
However, MSan unpoisoned the second parameter with the wrong size
`sizeof(__sanitizer_sigaction)`, inconsistent with its real type
`siginfo_t`.

This commit fixes this issue by correcting the size to
`sizeof(__sanitizer_siginfo)`.
---
 compiler-rt/lib/msan/msan_interceptors.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index 76255cdb742a..f94d3cb79aa0 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -1127,7 +1127,7 @@ static void SignalAction(int signo, void *si, void *uc) {
   SignalHandlerScope signal_handler_scope;
   ScopedThreadLocalStateBackup stlsb;
   UnpoisonParam(3);
-  __msan_unpoison(si, sizeof(__sanitizer_sigaction));
+  __msan_unpoison(si, sizeof(__sanitizer_siginfo));
   __msan_unpoison(uc, ucontext_t_sz(uc));
 
   typedef void (*sigaction_cb)(int, void *, void *);
-- 
cgit v1.2.3


From 4d71f20b287e398f10bbff55d52bec9683ef89d2 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler@amd.com>
Date: Wed, 18 Jun 2025 09:07:08 +0200
Subject: [GlobalISel] prevent G_UNMERGE_VALUES for vectors with different
 elements (#133335)

This commit prevents building a G_UNMERGE_VALUES instruction with
different source and destination vector elements in
`LegalizationArtifactCombiner::ArtifactValueFinder::tryCombineMergeLike()`,
e.g.:
`%1:_(<2 x s8>), %2:_(<2 x s8>) = G_UNMERGE_VALUES %0:_(<2 x s16>)`

This LLVM defect was identified via the AMD Fuzzing project.
---
 .../GlobalISel/LegalizationArtifactCombiner.h      |  5 +-
 .../CodeGen/AMDGPU/GlobalISel/insertelement.ll     | 55 ++++++++++++++++++++++
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 22f6a5fde546..8f560c42082f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -997,6 +997,7 @@ public:
 
       // Recognize UnmergeSrc that can be unmerged to DstTy directly.
       // Types have to be either both vector or both non-vector types.
+      // In case of vector types, the scalar elements need to match.
       // Merge-like opcodes are combined one at the time. First one creates new
       // unmerge, following should use the same unmerge (builder performs CSE).
       //
@@ -1005,7 +1006,9 @@ public:
       // %AnotherDst:_(DstTy) = G_merge_like_opcode %2:_(EltTy), %3
       //
       // %Dst:_(DstTy), %AnotherDst = G_UNMERGE_VALUES %UnmergeSrc
-      if ((DstTy.isVector() == UnmergeSrcTy.isVector()) &&
+      if (((!DstTy.isVector() && !UnmergeSrcTy.isVector()) ||
+           (DstTy.isVector() && UnmergeSrcTy.isVector() &&
+            DstTy.getScalarType() == UnmergeSrcTy.getScalarType())) &&
           (Elt0UnmergeIdx % NumMIElts == 0) &&
           getCoverTy(UnmergeSrcTy, DstTy) == UnmergeSrcTy) {
         if (!isSequenceFromUnmerge(MI, 0, Unmerge, Elt0UnmergeIdx, NumMIElts,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 8134eb3ca2af..132a89478c5f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -6506,3 +6506,58 @@ entry:
   %insert = insertelement <5 x double> %vec, double %val, i32 %idx
   ret <5 x double> %insert
 }
+
+; Found by fuzzer, reduced with llvm-reduce.
+define amdgpu_kernel void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
+; GPRIDX-LABEL: insert_very_small_from_very_large:
+; GPRIDX:       ; %bb.0: ; %bb
+; GPRIDX-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x0
+; GPRIDX-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x40
+; GPRIDX-NEXT:    s_waitcnt lgkmcnt(0)
+; GPRIDX-NEXT:    s_lshr_b32 s2, s12, 1
+; GPRIDX-NEXT:    s_and_b32 s2, s2, 1
+; GPRIDX-NEXT:    s_lshl_b32 s2, s2, 1
+; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
+; GPRIDX-NEXT:    v_mov_b32_e32 v2, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s1
+; GPRIDX-NEXT:    flat_store_byte v[0:1], v2
+; GPRIDX-NEXT:    s_endpgm
+;
+; GFX10-LABEL: insert_very_small_from_very_large:
+; GFX10:       ; %bb.0: ; %bb
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x0
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x40
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshr_b32 s2, s12, 1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s0
+; GFX10-NEXT:    s_and_b32 s2, s2, 1
+; GFX10-NEXT:    v_mov_b32_e32 v1, s1
+; GFX10-NEXT:    s_lshl_b32 s2, s2, 1
+; GFX10-NEXT:    v_mov_b32_e32 v2, s2
+; GFX10-NEXT:    flat_store_byte v[0:1], v2
+; GFX10-NEXT:    s_endpgm
+;
+; GFX11-LABEL: insert_very_small_from_very_large:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_load_b512 s[8:23], s[4:5], 0x0
+; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x40
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_lshr_b32 s2, s8, 1
+; GFX11-NEXT:    v_mov_b32_e32 v0, s0
+; GFX11-NEXT:    s_and_b32 s2, s2, 1
+; GFX11-NEXT:    v_mov_b32_e32 v1, s1
+; GFX11-NEXT:    s_lshl_b32 s2, s2, 1
+; GFX11-NEXT:    v_mov_b32_e32 v2, s2
+; GFX11-NEXT:    flat_store_b8 v[0:1], v2
+; GFX11-NEXT:    s_endpgm
+bb:
+  %a = bitcast <32 x i16> %L3 to i512
+  %b = trunc i512 %a to i8
+  %c = trunc i8 %b to i2
+  %d = bitcast i2 %c to <2 x i1>
+  %insert = insertelement <2 x i1> %d, i1 false, i32 0
+  store <2 x i1> %insert, ptr %ptr, align 1
+  ret void
+}
-- 
cgit v1.2.3


From 896e187a6e923b8441428f9db63c412d989fc51d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 17 Jun 2025 17:00:21 +0100
Subject: [X86] combineAndMaskToShift - pull out repeated SDLoc(). NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7f425b3d479d..335481b97d47 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -51225,7 +51225,8 @@ static SDValue combineBitOpWithPACK(unsigned Opc, const SDLoc &DL, EVT VT,
 /// If this is a zero/all-bits result that is bitwise-anded with a low bits
 /// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
 /// with a shift-right to eliminate loading the vector constant mask value.
-static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
+static SDValue combineAndMaskToShift(SDNode *N, const SDLoc &DL,
+                                     SelectionDAG &DAG,
                                      const X86Subtarget &Subtarget) {
   SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
   SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
@@ -51255,7 +51256,6 @@ static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
       Y = Op1;
     }
     if (X && Y) {
-      SDLoc DL(N);
       SDValue Sra =
           getTargetVShiftByConstNode(X86ISD::VSRAI, DL, VT.getSimpleVT(), X,
                                      VT.getScalarSizeInBits() - 1, DAG);
@@ -51278,7 +51278,6 @@ static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
   if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
     return SDValue();
 
-  SDLoc DL(N);
   unsigned ShiftVal = SplatVal.countr_one();
   SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
   SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT, Op0, ShAmt);
@@ -51845,7 +51844,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (SDValue R = combineAndNotIntoANDNP(N, DAG))
     return R;
 
-  if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
+  if (SDValue ShiftRight = combineAndMaskToShift(N, dl, DAG, Subtarget))
     return ShiftRight;
 
   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
-- 
cgit v1.2.3


From dac94f28e696e8234ec69bbed549533ea6b00227 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 17 Jun 2025 17:06:02 +0100
Subject: [X86] combineAndNotOrIntoAndNotAnd - pull out repeated SDLoc(). NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 335481b97d47..62912f1c8d12 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -51308,13 +51308,11 @@ static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
 
 /// Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z))
 /// This undoes the inverse fold performed in InstCombine
-static SDValue combineAndNotOrIntoAndNotAnd(SDNode *N, SelectionDAG &DAG) {
-
+static SDValue combineAndNotOrIntoAndNotAnd(SDNode *N, const SDLoc &DL,
+                                            SelectionDAG &DAG) {
   using namespace llvm::SDPatternMatch;
   MVT VT = N->getSimpleValueType(0);
-  SDLoc DL(N);
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-  if (!TLI.hasAndNot(SDValue(N, 0)))
+  if (!DAG.getTargetLoweringInfo().hasAndNot(SDValue(N, 0)))
     return SDValue();
 
   SDValue X, Y, Z;
@@ -51850,7 +51848,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
     return R;
 
-  if (SDValue R = combineAndNotOrIntoAndNotAnd(N, DAG))
+  if (SDValue R = combineAndNotOrIntoAndNotAnd(N, dl, DAG))
     return R;
 
   // fold (and (mul x, c1), c2) -> (mul x, (and c1, c2))
-- 
cgit v1.2.3


From 0875bee2b10185eca40aea3b3f49eb8462522eda Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 17 Jun 2025 17:14:13 +0100
Subject: [X86] combineAndNotIntoANDNP - pull out repeated SDLoc(). NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 62912f1c8d12..1ca5fc5376f0 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -50839,7 +50839,8 @@ static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
 }
 
 /// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
-static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
+static SDValue combineAndNotIntoANDNP(SDNode *N, const SDLoc &DL,
+                                      SelectionDAG &DAG) {
   assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDNP");
 
   MVT VT = N->getSimpleValueType(0);
@@ -50861,7 +50862,7 @@ static SDValue combineAndNotIntoANDNP(SDNode *N, SelectionDAG &DAG) {
 
   X = DAG.getBitcast(VT, X);
   Y = DAG.getBitcast(VT, Y);
-  return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
+  return DAG.getNode(X86ISD::ANDNP, DL, VT, X, Y);
 }
 
 /// Try to fold:
@@ -51839,7 +51840,7 @@ static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
   if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
     return R;
 
-  if (SDValue R = combineAndNotIntoANDNP(N, DAG))
+  if (SDValue R = combineAndNotIntoANDNP(N, dl ,DAG))
     return R;
 
   if (SDValue ShiftRight = combineAndMaskToShift(N, dl, DAG, Subtarget))
-- 
cgit v1.2.3


From 44b715293fcad79ef4a54474627ac574a759fa5a Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 17 Jun 2025 18:38:40 +0100
Subject: [PhaseOrdering][X86] Copy FMUL+ADDSUB/FMADDSUB build vector patterns
 from codegen tests

As detailed on #144489 - confirm the vectorisation of scalar FMUL+ADDSUB/FMADDSUB on various targets
---
 llvm/test/Transforms/PhaseOrdering/X86/fmaddsub.ll | 720 +++++++++++++++++++++
 1 file changed, 720 insertions(+)
 create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/fmaddsub.ll

diff --git a/llvm/test/Transforms/PhaseOrdering/X86/fmaddsub.ll b/llvm/test/Transforms/PhaseOrdering/X86/fmaddsub.ll
new file mode 100644
index 000000000000..ad4452431a48
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/fmaddsub.ll
@@ -0,0 +1,720 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=bdver2    | FileCheck %s --check-prefixes=CHECK,AVX,AVX_FMA4
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX_FMA3
+; RUN: opt < %s -O3 -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64    | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=CHECK,SSE,SSE4
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=bdver2    | FileCheck %s --check-prefixes=CHECK,AVX,AVX_FMA4
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,AVX,AVX_FMA3
+; RUN: opt < %s -passes="default<O3>" -S -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
+
+; This test checks the vectorisation of FMUL+ADDSUB/FMADDSUB patterns, including cases with undef elements.
+
+; Ideally, this should reach the backend with 1 fmul, 1 fsub, 1 fadd, and 1 shuffle.
+; That may require some coordination between VectorCombine, SLP, and other passes.
+
+define <4 x float> @buildvector_mul_addsub_ps128(<4 x float> %C, <4 x float> %D, <4 x float> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_addsub_ps128(
+; CHECK-NEXT:    [[A:%.*]] = fmul <4 x float> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <4 x float> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x float> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %A = fmul <4 x float> %C, %D
+  %A0 = extractelement <4 x float> %A, i32 0
+  %B0 = extractelement <4 x float> %B, i32 0
+  %sub0 = fsub float %A0, %B0
+  %A2 = extractelement <4 x float> %A, i32 2
+  %B2 = extractelement <4 x float> %B, i32 2
+  %sub2 = fsub float %A2, %B2
+  %A1 = extractelement <4 x float> %A, i32 1
+  %B1 = extractelement <4 x float> %B, i32 1
+  %add1 = fadd float %A1, %B1
+  %A3 = extractelement <4 x float> %A, i32 3
+  %B3 = extractelement <4 x float> %B, i32 3
+  %add3 = fadd float %A3, %B3
+  %vecinsert1 = insertelement <4 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <4 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <4 x float> %vecinsert3, float %add3, i32 3
+  ret <4 x float> %vecinsert4
+}
+
+define <2 x double> @buildvector_mul_addsub_pd128(<2 x double> %C, <2 x double> %D, <2 x double> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_addsub_pd128(
+; CHECK-NEXT:    [[A:%.*]] = fmul <2 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <2 x double> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <2 x double> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %A = fmul <2 x double> %C, %D
+  %A0 = extractelement <2 x double> %A, i32 0
+  %B0 = extractelement <2 x double> %B, i32 0
+  %sub0 = fsub double %A0, %B0
+  %A1 = extractelement <2 x double> %A, i32 1
+  %B1 = extractelement <2 x double> %B, i32 1
+  %add1 = fadd double %A1, %B1
+  %vecinsert1 = insertelement <2 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <2 x double> %vecinsert1, double %add1, i32 1
+  ret <2 x double> %vecinsert2
+}
+
+define <8 x float> @buildvector_mul_addsub_ps256(<8 x float> %C, <8 x float> %D, <8 x float> %B) #0 {
+; SSE2-LABEL: @buildvector_mul_addsub_ps256(
+; SSE2-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; SSE2-NEXT:    [[TMP0:%.*]] = fsub <8 x float> [[A]], [[B:%.*]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE2-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; SSE2-NEXT:    ret <8 x float> [[TMP4]]
+;
+; SSE4-LABEL: @buildvector_mul_addsub_ps256(
+; SSE4-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; SSE4-NEXT:    [[TMP0:%.*]] = fsub <8 x float> [[A]], [[B:%.*]]
+; SSE4-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE4-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; SSE4-NEXT:    ret <8 x float> [[TMP2]]
+;
+; AVX-LABEL: @buildvector_mul_addsub_ps256(
+; AVX-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX-NEXT:    [[TMP0:%.*]] = fsub <8 x float> [[A]], [[B:%.*]]
+; AVX-NEXT:    [[TMP1:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; AVX-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %A = fmul <8 x float> %C, %D
+  %A0 = extractelement <8 x float> %A, i32 0
+  %B0 = extractelement <8 x float> %B, i32 0
+  %sub0 = fsub float %A0, %B0
+  %A2 = extractelement <8 x float> %A, i32 2
+  %B2 = extractelement <8 x float> %B, i32 2
+  %sub2 = fsub float %A2, %B2
+  %A4 = extractelement <8 x float> %A, i32 4
+  %B4 = extractelement <8 x float> %B, i32 4
+  %sub4 = fsub float %A4, %B4
+  %A6 = extractelement <8 x float> %A, i32 6
+  %B6 = extractelement <8 x float> %B, i32 6
+  %sub6 = fsub float %A6, %B6
+  %A1 = extractelement <8 x float> %A, i32 1
+  %B1 = extractelement <8 x float> %B, i32 1
+  %add1 = fadd float %A1, %B1
+  %A3 = extractelement <8 x float> %A, i32 3
+  %B3 = extractelement <8 x float> %B, i32 3
+  %add3 = fadd float %A3, %B3
+  %A5 = extractelement <8 x float> %A, i32 5
+  %B5 = extractelement <8 x float> %B, i32 5
+  %add5 = fadd float %A5, %B5
+  %A7 = extractelement <8 x float> %A, i32 7
+  %B7 = extractelement <8 x float> %B, i32 7
+  %add7 = fadd float %A7, %B7
+  %vecinsert1 = insertelement <8 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <8 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <8 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <8 x float> %vecinsert3, float %add3, i32 3
+  %vecinsert5 = insertelement <8 x float> %vecinsert4, float %sub4, i32 4
+  %vecinsert6 = insertelement <8 x float> %vecinsert5, float %add5, i32 5
+  %vecinsert7 = insertelement <8 x float> %vecinsert6, float %sub6, i32 6
+  %vecinsert8 = insertelement <8 x float> %vecinsert7, float %add7, i32 7
+  ret <8 x float> %vecinsert8
+}
+
+define <4 x double> @buildvector_mul_addsub_pd256(<4 x double> %C, <4 x double> %D, <4 x double> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_addsub_pd256(
+; CHECK-NEXT:    [[A:%.*]] = fmul <4 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fsub <4 x double> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fadd <4 x double> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %A = fmul <4 x double> %C, %D
+  %A0 = extractelement <4 x double> %A, i32 0
+  %B0 = extractelement <4 x double> %B, i32 0
+  %sub0 = fsub double %A0, %B0
+  %A2 = extractelement <4 x double> %A, i32 2
+  %B2 = extractelement <4 x double> %B, i32 2
+  %sub2 = fsub double %A2, %B2
+  %A1 = extractelement <4 x double> %A, i32 1
+  %B1 = extractelement <4 x double> %B, i32 1
+  %add1 = fadd double %A1, %B1
+  %A3 = extractelement <4 x double> %A, i32 3
+  %B3 = extractelement <4 x double> %B, i32 3
+  %add3 = fadd double %A3, %B3
+  %vecinsert1 = insertelement <4 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <4 x double> %vecinsert1, double %add1, i32 1
+  %vecinsert3 = insertelement <4 x double> %vecinsert2, double %sub2, i32 2
+  %vecinsert4 = insertelement <4 x double> %vecinsert3, double %add3, i32 3
+  ret <4 x double> %vecinsert4
+}
+
+define <16 x float> @buildvector_mul_addsub_ps512(<16 x float> %C, <16 x float> %D, <16 x float> %B) #0 {
+; SSE-LABEL: @buildvector_mul_addsub_ps512(
+; SSE-NEXT:    [[A:%.*]] = fmul <16 x float> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[TMP0:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[B:%.*]], <16 x float> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13>
+; SSE-NEXT:    [[TMP2:%.*]] = fsub <12 x float> [[TMP0]], [[TMP1]]
+; SSE-NEXT:    [[TMP3:%.*]] = fadd <12 x float> [[TMP0]], [[TMP1]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <12 x float> [[TMP2]], <12 x float> [[TMP3]], <12 x i32> <i32 0, i32 13, i32 2, i32 15, i32 4, i32 5, i32 18, i32 7, i32 20, i32 9, i32 22, i32 23>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = fsub <2 x float> [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = fadd <2 x float> [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <12 x float> [[TMP4]], <12 x float> <float undef, float undef, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 12, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 13, i32 11, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> [[TMP8]], <16 x i32> <i32 0, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[VECINSERT161:%.*]] = shufflevector <16 x float> [[TMP9]], <16 x float> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; SSE-NEXT:    ret <16 x float> [[VECINSERT161]]
+;
+; AVX-LABEL: @buildvector_mul_addsub_ps512(
+; AVX-NEXT:    [[A:%.*]] = fmul <16 x float> [[C:%.*]], [[D:%.*]]
+; AVX-NEXT:    [[TMP0:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[B:%.*]], <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8>
+; AVX-NEXT:    [[TMP2:%.*]] = fsub <8 x float> [[TMP0]], [[TMP1]]
+; AVX-NEXT:    [[TMP3:%.*]] = fadd <8 x float> [[TMP0]], [[TMP1]]
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> [[TMP3]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 5, i32 14, i32 7>
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <4 x i32> <i32 9, i32 10, i32 11, i32 13>
+; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <4 x i32> <i32 9, i32 10, i32 11, i32 13>
+; AVX-NEXT:    [[TMP7:%.*]] = fadd <4 x float> [[TMP5]], [[TMP6]]
+; AVX-NEXT:    [[TMP8:%.*]] = fsub <4 x float> [[TMP5]], [[TMP6]]
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:    [[TMP11:%.*]] = fsub <2 x float> [[TMP9]], [[TMP10]]
+; AVX-NEXT:    [[TMP12:%.*]] = fadd <2 x float> [[TMP9]], [[TMP10]]
+; AVX-NEXT:    [[TMP13:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> <float undef, float undef, float poison, float poison, float poison, float poison, float poison, float poison>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 9, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP14:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP8]], <16 x i32> <i32 0, i32 5, i32 2, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[VECINSERT141:%.*]] = shufflevector <16 x float> [[TMP13]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 16, i32 17, i32 18, i32 12, i32 20, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP15:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP12]], <16 x i32> <i32 0, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[VECINSERT162:%.*]] = shufflevector <16 x float> [[VECINSERT141]], <16 x float> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; AVX-NEXT:    ret <16 x float> [[VECINSERT162]]
+;
+  %A = fmul <16 x float> %C, %D
+  %A0 = extractelement <16 x float> %A, i32 0
+  %B0 = extractelement <16 x float> %B, i32 0
+  %sub0 = fsub float %A0, %B0
+  %A2 = extractelement <16 x float> %A, i32 2
+  %B2 = extractelement <16 x float> %B, i32 2
+  %sub2 = fsub float %A2, %B2
+  %A4 = extractelement <16 x float> %A, i32 4
+  %B4 = extractelement <16 x float> %B, i32 4
+  %sub4 = fsub float %A4, %B4
+  %A6 = extractelement <16 x float> %A, i32 6
+  %B6 = extractelement <16 x float> %B, i32 6
+  %sub6 = fsub float %A6, %B6
+  %A8 = extractelement <16 x float> %A, i32 8
+  %B8 = extractelement <16 x float> %B, i32 8
+  %sub8 = fsub float %A8, %B8
+  %A10 = extractelement <16 x float> %A, i32 10
+  %B10 = extractelement <16 x float> %B, i32 10
+  %sub10 = fsub float %A10, %B10
+  %A12 = extractelement <16 x float> %A, i32 12
+  %B12 = extractelement <16 x float> %B, i32 12
+  %sub12 = fsub float %A12, %B12
+  %A14 = extractelement <16 x float> %A, i32 14
+  %B14 = extractelement <16 x float> %B, i32 14
+  %sub14 = fsub float %A14, %B14
+  %A1 = extractelement <16 x float> %A, i32 1
+  %B1 = extractelement <16 x float> %B, i32 1
+  %add1 = fadd float %A1, %B1
+  %A3 = extractelement <16 x float> %A, i32 3
+  %B3 = extractelement <16 x float> %B, i32 3
+  %add3 = fadd float %A3, %B3
+  %A5 = extractelement <16 x float> %A, i32 5
+  %B5 = extractelement <16 x float> %B, i32 5
+  %add5 = fadd float %A5, %B5
+  %A7 = extractelement <16 x float> %A, i32 7
+  %B7 = extractelement <16 x float> %B, i32 7
+  %add7 = fadd float %A7, %B7
+  %A9 = extractelement <16 x float> %A, i32 9
+  %B9 = extractelement <16 x float> %B, i32 9
+  %add9 = fadd float %A9, %B9
+  %A11 = extractelement <16 x float> %A, i32 11
+  %B11 = extractelement <16 x float> %B, i32 11
+  %add11 = fadd float %A11, %B11
+  %A13 = extractelement <16 x float> %A, i32 13
+  %B13 = extractelement <16 x float> %B, i32 13
+  %add13 = fadd float %A13, %B13
+  %A15 = extractelement <16 x float> %A, i32 15
+  %B15 = extractelement <16 x float> %B, i32 15
+  %add15 = fadd float %A15, %B15
+  %vecinsert1 = insertelement <16 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <16 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <16 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <16 x float> %vecinsert3, float %add3, i32 3
+  %vecinsert5 = insertelement <16 x float> %vecinsert4, float %sub4, i32 4
+  ; element 5 is undef
+  %vecinsert7 = insertelement <16 x float> %vecinsert5, float %sub6, i32 6
+  %vecinsert8 = insertelement <16 x float> %vecinsert7, float %add7, i32 7
+  %vecinsert9 = insertelement <16 x float> %vecinsert8, float %sub8, i32 8
+  %vecinsert10 = insertelement <16 x float> %vecinsert9, float %add9, i32 9
+  %vecinsert11 = insertelement <16 x float> %vecinsert10, float %sub10, i32 10
+  %vecinsert12 = insertelement <16 x float> %vecinsert11, float %add11, i32 11
+  ; element 12 is undef
+  %vecinsert14 = insertelement <16 x float> %vecinsert12, float %add13, i32 13
+  %vecinsert15 = insertelement <16 x float> %vecinsert14, float %sub14, i32 14
+  %vecinsert16 = insertelement <16 x float> %vecinsert15, float %add15, i32 15
+  ret <16 x float> %vecinsert16
+}
+
+define <8 x double> @buildvector_mul_addsub_pd512(<8 x double> %C, <8 x double> %D, <8 x double> %B) #0 {
+; SSE-LABEL: @buildvector_mul_addsub_pd512(
+; SSE-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[TMP0:%.*]] = fsub <8 x double> [[A]], [[B:%.*]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[TMP0]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE-NEXT:    [[TMP2:%.*]] = fadd <8 x double> [[A]], [[B]]
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <6 x double> [[TMP4]], <6 x double> [[TMP5]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i64 7
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x double> [[B]], i64 7
+; SSE-NEXT:    [[ADD7:%.*]] = fadd double [[A7]], [[B7]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <6 x double> [[TMP6]], <6 x double> <double undef, double poison, double poison, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 poison>
+; SSE-NEXT:    [[VECINSERT8:%.*]] = insertelement <8 x double> [[TMP7]], double [[ADD7]], i64 7
+; SSE-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX_FMA4-LABEL: @buildvector_mul_addsub_pd512(
+; AVX_FMA4-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX_FMA4-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA4-NEXT:    [[TMP2:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA4-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX_FMA4-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i64 7
+; AVX_FMA4-NEXT:    [[B7:%.*]] = extractelement <8 x double> [[B]], i64 7
+; AVX_FMA4-NEXT:    [[ADD7:%.*]] = fadd double [[A7]], [[B7]]
+; AVX_FMA4-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[TMP6:%.*]] = fsub <8 x double> [[A]], [[B]]
+; AVX_FMA4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX_FMA4-NEXT:    [[VECINSERT8:%.*]] = insertelement <8 x double> [[VECINSERT71]], double [[ADD7]], i64 7
+; AVX_FMA4-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX_FMA3-LABEL: @buildvector_mul_addsub_pd512(
+; AVX_FMA3-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX_FMA3-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA3-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA3-NEXT:    [[TMP2:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA3-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA3-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX_FMA3-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[A]], [[B]]
+; AVX_FMA3-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX_FMA3-NEXT:    [[TMP7:%.*]] = fsub <8 x double> [[A]], [[B]]
+; AVX_FMA3-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA3-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX_FMA3-NEXT:    [[VECINSERT8:%.*]] = shufflevector <8 x double> [[VECINSERT71]], <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; AVX_FMA3-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX512-LABEL: @buildvector_mul_addsub_pd512(
+; AVX512-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP2:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX512-NEXT:    [[TMP3:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX512-NEXT:    [[TMP5:%.*]] = fadd <8 x double> [[A]], [[B]]
+; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP7:%.*]] = fsub <8 x double> [[A]], [[B]]
+; AVX512-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX512-NEXT:    [[VECINSERT8:%.*]] = shufflevector <8 x double> [[VECINSERT71]], <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; AVX512-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+  %A = fmul <8 x double> %C, %D
+  %A0 = extractelement <8 x double> %A, i32 0
+  %B0 = extractelement <8 x double> %B, i32 0
+  %sub0 = fsub double %A0, %B0
+  %A2 = extractelement <8 x double> %A, i32 2
+  %B2 = extractelement <8 x double> %B, i32 2
+  %sub2 = fsub double %A2, %B2
+  %A4 = extractelement <8 x double> %A, i32 4
+  %B4 = extractelement <8 x double> %B, i32 4
+  %sub4 = fsub double %A4, %B4
+  %A6 = extractelement <8 x double> %A, i32 6
+  %B6 = extractelement <8 x double> %B, i32 6
+  %sub6 = fsub double %A6, %B6
+  %A1 = extractelement <8 x double> %A, i32 1
+  %B1 = extractelement <8 x double> %B, i32 1
+  %add1 = fadd double %A1, %B1
+  %A3 = extractelement <8 x double> %A, i32 3
+  %B3 = extractelement <8 x double> %B, i32 3
+  %add3 = fadd double %A3, %B3
+  %A7 = extractelement <8 x double> %A, i32 7
+  %B7 = extractelement <8 x double> %B, i32 7
+  %add7 = fadd double %A7, %B7
+  %vecinsert1 = insertelement <8 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <8 x double> %vecinsert1, double %add1, i32 1
+  %vecinsert3 = insertelement <8 x double> %vecinsert2, double %sub2, i32 2
+  %vecinsert4 = insertelement <8 x double> %vecinsert3, double %add3, i32 3
+  %vecinsert5 = insertelement <8 x double> %vecinsert4, double %sub4, i32 4
+  ; element 5 is undef
+  %vecinsert7 = insertelement <8 x double> %vecinsert5, double %sub6, i32 6
+  %vecinsert8 = insertelement <8 x double> %vecinsert7, double %add7, i32 7
+  ret <8 x double> %vecinsert8
+}
+
+define <4 x float> @buildvector_mul_subadd_ps128(<4 x float> %C, <4 x float> %D, <4 x float> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_subadd_ps128(
+; CHECK-NEXT:    [[A:%.*]] = fmul <4 x float> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <4 x float> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x float> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x float> [[TMP0]], <4 x float> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x float> [[TMP2]]
+;
+  %A = fmul <4 x float> %C, %D
+  %A0 = extractelement <4 x float> %A, i32 0
+  %B0 = extractelement <4 x float> %B, i32 0
+  %sub0 = fadd float %A0, %B0
+  %A2 = extractelement <4 x float> %A, i32 2
+  %B2 = extractelement <4 x float> %B, i32 2
+  %sub2 = fadd float %A2, %B2
+  %A1 = extractelement <4 x float> %A, i32 1
+  %B1 = extractelement <4 x float> %B, i32 1
+  %add1 = fsub float %A1, %B1
+  %A3 = extractelement <4 x float> %A, i32 3
+  %B3 = extractelement <4 x float> %B, i32 3
+  %add3 = fsub float %A3, %B3
+  %vecinsert1 = insertelement <4 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <4 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <4 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <4 x float> %vecinsert3, float %add3, i32 3
+  ret <4 x float> %vecinsert4
+}
+
+define <2 x double> @buildvector_mul_subadd_pd128(<2 x double> %C, <2 x double> %D, <2 x double> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_subadd_pd128(
+; CHECK-NEXT:    [[A:%.*]] = fmul <2 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <2 x double> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <2 x double> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> [[TMP1]], <2 x i32> <i32 0, i32 3>
+; CHECK-NEXT:    ret <2 x double> [[TMP2]]
+;
+  %A = fmul <2 x double> %C, %D
+  %A0 = extractelement <2 x double> %A, i32 0
+  %B0 = extractelement <2 x double> %B, i32 0
+  %sub0 = fadd double %A0, %B0
+  %A1 = extractelement <2 x double> %A, i32 1
+  %B1 = extractelement <2 x double> %B, i32 1
+  %add1 = fsub double %A1, %B1
+  %vecinsert1 = insertelement <2 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <2 x double> %vecinsert1, double %add1, i32 1
+  ret <2 x double> %vecinsert2
+}
+
+define <8 x float> @buildvector_mul_subadd_ps256(<8 x float> %C, <8 x float> %D, <8 x float> %B) #0 {
+; SSE2-LABEL: @buildvector_mul_subadd_ps256(
+; SSE2-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; SSE2-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A]], [[B:%.*]]
+; SSE2-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE2-NEXT:    [[TMP2:%.*]] = fsub <8 x float> [[A]], [[B]]
+; SSE2-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE2-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> [[TMP3]], <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; SSE2-NEXT:    ret <8 x float> [[TMP4]]
+;
+; SSE4-LABEL: @buildvector_mul_subadd_ps256(
+; SSE4-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; SSE4-NEXT:    [[TMP0:%.*]] = fsub <8 x float> [[A]], [[B:%.*]]
+; SSE4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; SSE4-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[A]], [[B]]
+; SSE4-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; SSE4-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; SSE4-NEXT:    ret <8 x float> [[TMP6]]
+;
+; AVX_FMA4-LABEL: @buildvector_mul_subadd_ps256(
+; AVX_FMA4-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX_FMA4-NEXT:    [[TMP0:%.*]] = fsub <8 x float> [[A]], [[B:%.*]]
+; AVX_FMA4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX_FMA4-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[A]], [[B]]
+; AVX_FMA4-NEXT:    [[TMP3:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[TMP5:%.*]] = shufflevector <8 x float> [[TMP3]], <8 x float> [[TMP4]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
+; AVX_FMA4-NEXT:    [[TMP6:%.*]] = shufflevector <8 x float> [[TMP5]], <8 x float> poison, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
+; AVX_FMA4-NEXT:    ret <8 x float> [[TMP6]]
+;
+; AVX_FMA3-LABEL: @buildvector_mul_subadd_ps256(
+; AVX_FMA3-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX_FMA3-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A]], [[B:%.*]]
+; AVX_FMA3-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A]], [[B]]
+; AVX_FMA3-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; AVX_FMA3-NEXT:    ret <8 x float> [[TMP2]]
+;
+; AVX512-LABEL: @buildvector_mul_subadd_ps256(
+; AVX512-NEXT:    [[A:%.*]] = fmul <8 x float> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[TMP0:%.*]] = fadd <8 x float> [[A]], [[B:%.*]]
+; AVX512-NEXT:    [[TMP1:%.*]] = fsub <8 x float> [[A]], [[B]]
+; AVX512-NEXT:    [[TMP2:%.*]] = shufflevector <8 x float> [[TMP0]], <8 x float> [[TMP1]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 13, i32 6, i32 15>
+; AVX512-NEXT:    ret <8 x float> [[TMP2]]
+;
+  %A = fmul <8 x float> %C, %D
+  %A0 = extractelement <8 x float> %A, i32 0
+  %B0 = extractelement <8 x float> %B, i32 0
+  %sub0 = fadd float %A0, %B0
+  %A2 = extractelement <8 x float> %A, i32 2
+  %B2 = extractelement <8 x float> %B, i32 2
+  %sub2 = fadd float %A2, %B2
+  %A4 = extractelement <8 x float> %A, i32 4
+  %B4 = extractelement <8 x float> %B, i32 4
+  %sub4 = fadd float %A4, %B4
+  %A6 = extractelement <8 x float> %A, i32 6
+  %B6 = extractelement <8 x float> %B, i32 6
+  %sub6 = fadd float %A6, %B6
+  %A1 = extractelement <8 x float> %A, i32 1
+  %B1 = extractelement <8 x float> %B, i32 1
+  %add1 = fsub float %A1, %B1
+  %A3 = extractelement <8 x float> %A, i32 3
+  %B3 = extractelement <8 x float> %B, i32 3
+  %add3 = fsub float %A3, %B3
+  %A5 = extractelement <8 x float> %A, i32 5
+  %B5 = extractelement <8 x float> %B, i32 5
+  %add5 = fsub float %A5, %B5
+  %A7 = extractelement <8 x float> %A, i32 7
+  %B7 = extractelement <8 x float> %B, i32 7
+  %add7 = fsub float %A7, %B7
+  %vecinsert1 = insertelement <8 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <8 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <8 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <8 x float> %vecinsert3, float %add3, i32 3
+  %vecinsert5 = insertelement <8 x float> %vecinsert4, float %sub4, i32 4
+  %vecinsert6 = insertelement <8 x float> %vecinsert5, float %add5, i32 5
+  %vecinsert7 = insertelement <8 x float> %vecinsert6, float %sub6, i32 6
+  %vecinsert8 = insertelement <8 x float> %vecinsert7, float %add7, i32 7
+  ret <8 x float> %vecinsert8
+}
+
+define <4 x double> @buildvector_mul_subadd_pd256(<4 x double> %C, <4 x double> %D, <4 x double> %B) #0 {
+; CHECK-LABEL: @buildvector_mul_subadd_pd256(
+; CHECK-NEXT:    [[A:%.*]] = fmul <4 x double> [[C:%.*]], [[D:%.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = fadd <4 x double> [[A]], [[B:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = fsub <4 x double> [[A]], [[B]]
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> [[TMP1]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; CHECK-NEXT:    ret <4 x double> [[TMP2]]
+;
+  %A = fmul <4 x double> %C, %D
+  %A0 = extractelement <4 x double> %A, i32 0
+  %B0 = extractelement <4 x double> %B, i32 0
+  %sub0 = fadd double %A0, %B0
+  %A2 = extractelement <4 x double> %A, i32 2
+  %B2 = extractelement <4 x double> %B, i32 2
+  %sub2 = fadd double %A2, %B2
+  %A1 = extractelement <4 x double> %A, i32 1
+  %B1 = extractelement <4 x double> %B, i32 1
+  %add1 = fsub double %A1, %B1
+  %A3 = extractelement <4 x double> %A, i32 3
+  %B3 = extractelement <4 x double> %B, i32 3
+  %add3 = fsub double %A3, %B3
+  %vecinsert1 = insertelement <4 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <4 x double> %vecinsert1, double %add1, i32 1
+  %vecinsert3 = insertelement <4 x double> %vecinsert2, double %sub2, i32 2
+  %vecinsert4 = insertelement <4 x double> %vecinsert3, double %add3, i32 3
+  ret <4 x double> %vecinsert4
+}
+
+define <16 x float> @buildvector_mul_subadd_ps512(<16 x float> %C, <16 x float> %D, <16 x float> %B) #0 {
+; SSE-LABEL: @buildvector_mul_subadd_ps512(
+; SSE-NEXT:    [[A:%.*]] = fmul <16 x float> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[TMP0:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13>
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[B:%.*]], <16 x float> poison, <12 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 13>
+; SSE-NEXT:    [[TMP2:%.*]] = fadd <12 x float> [[TMP0]], [[TMP1]]
+; SSE-NEXT:    [[TMP3:%.*]] = fsub <12 x float> [[TMP0]], [[TMP1]]
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <12 x float> [[TMP2]], <12 x float> [[TMP3]], <12 x i32> <i32 0, i32 13, i32 2, i32 15, i32 4, i32 5, i32 18, i32 7, i32 20, i32 9, i32 22, i32 23>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; SSE-NEXT:    [[TMP7:%.*]] = fadd <2 x float> [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP8:%.*]] = fsub <2 x float> [[TMP5]], [[TMP6]]
+; SSE-NEXT:    [[TMP9:%.*]] = shufflevector <12 x float> [[TMP4]], <12 x float> <float undef, float undef, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison, float poison>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 12, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 13, i32 11, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP10:%.*]] = shufflevector <2 x float> [[TMP7]], <2 x float> [[TMP8]], <16 x i32> <i32 0, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[VECINSERT161:%.*]] = shufflevector <16 x float> [[TMP9]], <16 x float> [[TMP10]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; SSE-NEXT:    ret <16 x float> [[VECINSERT161]]
+;
+; AVX-LABEL: @buildvector_mul_subadd_ps512(
+; AVX-NEXT:    [[A:%.*]] = fmul <16 x float> [[C:%.*]], [[D:%.*]]
+; AVX-NEXT:    [[TMP0:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8>
+; AVX-NEXT:    [[TMP1:%.*]] = shufflevector <16 x float> [[B:%.*]], <16 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 6, i32 7, i32 8>
+; AVX-NEXT:    [[TMP2:%.*]] = fadd <8 x float> [[TMP0]], [[TMP1]]
+; AVX-NEXT:    [[TMP3:%.*]] = fsub <8 x float> [[TMP0]], [[TMP1]]
+; AVX-NEXT:    [[TMP4:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> [[TMP3]], <8 x i32> <i32 0, i32 9, i32 2, i32 11, i32 4, i32 5, i32 14, i32 7>
+; AVX-NEXT:    [[TMP5:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <4 x i32> <i32 9, i32 10, i32 11, i32 13>
+; AVX-NEXT:    [[TMP6:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <4 x i32> <i32 9, i32 10, i32 11, i32 13>
+; AVX-NEXT:    [[TMP7:%.*]] = fsub <4 x float> [[TMP5]], [[TMP6]]
+; AVX-NEXT:    [[TMP8:%.*]] = fadd <4 x float> [[TMP5]], [[TMP6]]
+; AVX-NEXT:    [[TMP9:%.*]] = shufflevector <16 x float> [[A]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:    [[TMP10:%.*]] = shufflevector <16 x float> [[B]], <16 x float> poison, <2 x i32> <i32 14, i32 15>
+; AVX-NEXT:    [[TMP11:%.*]] = fadd <2 x float> [[TMP9]], [[TMP10]]
+; AVX-NEXT:    [[TMP12:%.*]] = fsub <2 x float> [[TMP9]], [[TMP10]]
+; AVX-NEXT:    [[TMP13:%.*]] = shufflevector <8 x float> [[TMP4]], <8 x float> <float undef, float undef, float poison, float poison, float poison, float poison, float poison, float poison>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 8, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 9, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP14:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> [[TMP8]], <16 x i32> <i32 0, i32 5, i32 2, i32 poison, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[VECINSERT141:%.*]] = shufflevector <16 x float> [[TMP13]], <16 x float> [[TMP14]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 16, i32 17, i32 18, i32 12, i32 20, i32 poison, i32 poison>
+; AVX-NEXT:    [[TMP15:%.*]] = shufflevector <2 x float> [[TMP11]], <2 x float> [[TMP12]], <16 x i32> <i32 0, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX-NEXT:    [[VECINSERT162:%.*]] = shufflevector <16 x float> [[VECINSERT141]], <16 x float> [[TMP15]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 16, i32 17>
+; AVX-NEXT:    ret <16 x float> [[VECINSERT162]]
+;
+  %A = fmul <16 x float> %C, %D
+  %A0 = extractelement <16 x float> %A, i32 0
+  %B0 = extractelement <16 x float> %B, i32 0
+  %sub0 = fadd float %A0, %B0
+  %A2 = extractelement <16 x float> %A, i32 2
+  %B2 = extractelement <16 x float> %B, i32 2
+  %sub2 = fadd float %A2, %B2
+  %A4 = extractelement <16 x float> %A, i32 4
+  %B4 = extractelement <16 x float> %B, i32 4
+  %sub4 = fadd float %A4, %B4
+  %A6 = extractelement <16 x float> %A, i32 6
+  %B6 = extractelement <16 x float> %B, i32 6
+  %sub6 = fadd float %A6, %B6
+  %A8 = extractelement <16 x float> %A, i32 8
+  %B8 = extractelement <16 x float> %B, i32 8
+  %sub8 = fadd float %A8, %B8
+  %A10 = extractelement <16 x float> %A, i32 10
+  %B10 = extractelement <16 x float> %B, i32 10
+  %sub10 = fadd float %A10, %B10
+  %A12 = extractelement <16 x float> %A, i32 12
+  %B12 = extractelement <16 x float> %B, i32 12
+  %sub12 = fadd float %A12, %B12
+  %A14 = extractelement <16 x float> %A, i32 14
+  %B14 = extractelement <16 x float> %B, i32 14
+  %sub14 = fadd float %A14, %B14
+  %A1 = extractelement <16 x float> %A, i32 1
+  %B1 = extractelement <16 x float> %B, i32 1
+  %add1 = fsub float %A1, %B1
+  %A3 = extractelement <16 x float> %A, i32 3
+  %B3 = extractelement <16 x float> %B, i32 3
+  %add3 = fsub float %A3, %B3
+  %A5 = extractelement <16 x float> %A, i32 5
+  %B5 = extractelement <16 x float> %B, i32 5
+  %add5 = fsub float %A5, %B5
+  %A7 = extractelement <16 x float> %A, i32 7
+  %B7 = extractelement <16 x float> %B, i32 7
+  %add7 = fsub float %A7, %B7
+  %A9 = extractelement <16 x float> %A, i32 9
+  %B9 = extractelement <16 x float> %B, i32 9
+  %add9 = fsub float %A9, %B9
+  %A11 = extractelement <16 x float> %A, i32 11
+  %B11 = extractelement <16 x float> %B, i32 11
+  %add11 = fsub float %A11, %B11
+  %A13 = extractelement <16 x float> %A, i32 13
+  %B13 = extractelement <16 x float> %B, i32 13
+  %add13 = fsub float %A13, %B13
+  %A15 = extractelement <16 x float> %A, i32 15
+  %B15 = extractelement <16 x float> %B, i32 15
+  %add15 = fsub float %A15, %B15
+  %vecinsert1 = insertelement <16 x float> undef, float %sub0, i32 0
+  %vecinsert2 = insertelement <16 x float> %vecinsert1, float %add1, i32 1
+  %vecinsert3 = insertelement <16 x float> %vecinsert2, float %sub2, i32 2
+  %vecinsert4 = insertelement <16 x float> %vecinsert3, float %add3, i32 3
+  %vecinsert5 = insertelement <16 x float> %vecinsert4, float %sub4, i32 4
+  ; element 5 is undef
+  %vecinsert7 = insertelement <16 x float> %vecinsert5, float %sub6, i32 6
+  %vecinsert8 = insertelement <16 x float> %vecinsert7, float %add7, i32 7
+  %vecinsert9 = insertelement <16 x float> %vecinsert8, float %sub8, i32 8
+  %vecinsert10 = insertelement <16 x float> %vecinsert9, float %add9, i32 9
+  %vecinsert11 = insertelement <16 x float> %vecinsert10, float %sub10, i32 10
+  %vecinsert12 = insertelement <16 x float> %vecinsert11, float %add11, i32 11
+  ; element 12 is undef
+  %vecinsert14 = insertelement <16 x float> %vecinsert12, float %add13, i32 13
+  %vecinsert15 = insertelement <16 x float> %vecinsert14, float %sub14, i32 14
+  %vecinsert16 = insertelement <16 x float> %vecinsert15, float %add15, i32 15
+  ret <16 x float> %vecinsert16
+}
+
+define <8 x double> @buildvector_mul_subadd_pd512(<8 x double> %C, <8 x double> %D, <8 x double> %B) #0 {
+; SSE-LABEL: @buildvector_mul_subadd_pd512(
+; SSE-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; SSE-NEXT:    [[TMP0:%.*]] = fadd <8 x double> [[A]], [[B:%.*]]
+; SSE-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[TMP0]], <8 x double> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; SSE-NEXT:    [[TMP2:%.*]] = fsub <8 x double> [[A]], [[B]]
+; SSE-NEXT:    [[TMP3:%.*]] = shufflevector <8 x double> [[TMP2]], <8 x double> poison, <2 x i32> <i32 1, i32 3>
+; SSE-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP1]], <4 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP5:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <6 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE-NEXT:    [[TMP6:%.*]] = shufflevector <6 x double> [[TMP4]], <6 x double> [[TMP5]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 6, i32 7>
+; SSE-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i64 7
+; SSE-NEXT:    [[B7:%.*]] = extractelement <8 x double> [[B]], i64 7
+; SSE-NEXT:    [[ADD7:%.*]] = fsub double [[A7]], [[B7]]
+; SSE-NEXT:    [[TMP7:%.*]] = shufflevector <6 x double> [[TMP6]], <6 x double> <double undef, double poison, double poison, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 poison>
+; SSE-NEXT:    [[VECINSERT8:%.*]] = insertelement <8 x double> [[TMP7]], double [[ADD7]], i64 7
+; SSE-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX_FMA4-LABEL: @buildvector_mul_subadd_pd512(
+; AVX_FMA4-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX_FMA4-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA4-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA4-NEXT:    [[TMP2:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA4-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA4-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX_FMA4-NEXT:    [[A7:%.*]] = extractelement <8 x double> [[A]], i64 7
+; AVX_FMA4-NEXT:    [[B7:%.*]] = extractelement <8 x double> [[B]], i64 7
+; AVX_FMA4-NEXT:    [[ADD7:%.*]] = fsub double [[A7]], [[B7]]
+; AVX_FMA4-NEXT:    [[TMP5:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[TMP6:%.*]] = fadd <8 x double> [[A]], [[B]]
+; AVX_FMA4-NEXT:    [[TMP7:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA4-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP5]], <8 x double> [[TMP7]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX_FMA4-NEXT:    [[VECINSERT8:%.*]] = insertelement <8 x double> [[VECINSERT71]], double [[ADD7]], i64 7
+; AVX_FMA4-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX_FMA3-LABEL: @buildvector_mul_subadd_pd512(
+; AVX_FMA3-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX_FMA3-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA3-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX_FMA3-NEXT:    [[TMP2:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA3-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX_FMA3-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX_FMA3-NEXT:    [[TMP5:%.*]] = fsub <8 x double> [[A]], [[B]]
+; AVX_FMA3-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX_FMA3-NEXT:    [[TMP7:%.*]] = fadd <8 x double> [[A]], [[B]]
+; AVX_FMA3-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX_FMA3-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX_FMA3-NEXT:    [[VECINSERT8:%.*]] = shufflevector <8 x double> [[VECINSERT71]], <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; AVX_FMA3-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+; AVX512-LABEL: @buildvector_mul_subadd_pd512(
+; AVX512-NEXT:    [[A:%.*]] = fmul <8 x double> [[C:%.*]], [[D:%.*]]
+; AVX512-NEXT:    [[TMP0:%.*]] = shufflevector <8 x double> [[A]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP1:%.*]] = shufflevector <8 x double> [[B:%.*]], <8 x double> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; AVX512-NEXT:    [[TMP2:%.*]] = fadd <4 x double> [[TMP0]], [[TMP1]]
+; AVX512-NEXT:    [[TMP3:%.*]] = fsub <4 x double> [[TMP0]], [[TMP1]]
+; AVX512-NEXT:    [[TMP4:%.*]] = shufflevector <4 x double> [[TMP2]], <4 x double> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7>
+; AVX512-NEXT:    [[TMP5:%.*]] = fsub <8 x double> [[A]], [[B]]
+; AVX512-NEXT:    [[TMP6:%.*]] = shufflevector <4 x double> [[TMP4]], <4 x double> <double undef, double poison, double poison, double poison>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 4, i32 poison, i32 poison>
+; AVX512-NEXT:    [[TMP7:%.*]] = fadd <8 x double> [[A]], [[B]]
+; AVX512-NEXT:    [[TMP8:%.*]] = shufflevector <8 x double> [[TMP7]], <8 x double> poison, <8 x i32> <i32 4, i32 poison, i32 6, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT:    [[VECINSERT71:%.*]] = shufflevector <8 x double> [[TMP6]], <8 x double> [[TMP8]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 5, i32 10, i32 poison>
+; AVX512-NEXT:    [[VECINSERT8:%.*]] = shufflevector <8 x double> [[VECINSERT71]], <8 x double> [[TMP5]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 15>
+; AVX512-NEXT:    ret <8 x double> [[VECINSERT8]]
+;
+  %A = fmul <8 x double> %C, %D
+  %A0 = extractelement <8 x double> %A, i32 0
+  %B0 = extractelement <8 x double> %B, i32 0
+  %sub0 = fadd double %A0, %B0
+  %A2 = extractelement <8 x double> %A, i32 2
+  %B2 = extractelement <8 x double> %B, i32 2
+  %sub2 = fadd double %A2, %B2
+  %A4 = extractelement <8 x double> %A, i32 4
+  %B4 = extractelement <8 x double> %B, i32 4
+  %sub4 = fadd double %A4, %B4
+  %A6 = extractelement <8 x double> %A, i32 6
+  %B6 = extractelement <8 x double> %B, i32 6
+  %sub6 = fadd double %A6, %B6
+  %A1 = extractelement <8 x double> %A, i32 1
+  %B1 = extractelement <8 x double> %B, i32 1
+  %add1 = fsub double %A1, %B1
+  %A3 = extractelement <8 x double> %A, i32 3
+  %B3 = extractelement <8 x double> %B, i32 3
+  %add3 = fsub double %A3, %B3
+  %A7 = extractelement <8 x double> %A, i32 7
+  %B7 = extractelement <8 x double> %B, i32 7
+  %add7 = fsub double %A7, %B7
+  %vecinsert1 = insertelement <8 x double> undef, double %sub0, i32 0
+  %vecinsert2 = insertelement <8 x double> %vecinsert1, double %add1, i32 1
+  %vecinsert3 = insertelement <8 x double> %vecinsert2, double %sub2, i32 2
+  %vecinsert4 = insertelement <8 x double> %vecinsert3, double %add3, i32 3
+  %vecinsert5 = insertelement <8 x double> %vecinsert4, double %sub4, i32 4
+  ; element 5 is undef
+  %vecinsert7 = insertelement <8 x double> %vecinsert5, double %sub6, i32 6
+  %vecinsert8 = insertelement <8 x double> %vecinsert7, double %add7, i32 7
+  ret <8 x double> %vecinsert8
+}
+
+attributes #0 = { nounwind "unsafe-fp-math"="true" }
-- 
cgit v1.2.3


From 45ea46c44636094e9fcdbbeabfd11f9d0fad5e38 Mon Sep 17 00:00:00 2001
From: Garvit Gupta <quic_garvgupt@quicinc.com>
Date: Wed, 18 Jun 2025 12:50:48 +0530
Subject: Reland [Driver] Add support for GCC installation detection in
 Baremetal toolchain (#144640)

This patch introduces enhancements to the Baremetal toolchain to support
GCC toolchain detection.
- If the --gcc-install-dir or --gcc-toolchain options are provided and
point to valid paths, the sysroot is derived from those locations.
- If not, the logic falls back to the existing sysroot inference
mechanism already present in the Baremetal toolchain.
- Support for adding include paths for the libstdc++ library has also
been added.

Additionally, the restriction to always use the integrated assembler has
been removed. With a valid GCC installation, the GNU assembler can now
be used as well.

This patch currently updates and adds tests for the ARM target only.
RISC-V-specific tests will be introduced in a later patch, once the
RISCVToolChain is fully merged into the Baremetal toolchain. At this
stage, there is no way to test the RISC-V target within this PR.

RFC:

https://discourse.llvm.org/t/merging-riscvtoolchain-and-baremetal-toolchains/75524
---
 clang/docs/Toolchain.rst                           |   5 +
 clang/include/clang/Basic/DiagnosticDriverKinds.td |   3 +
 clang/lib/Driver/ToolChains/BareMetal.cpp          | 235 +++++++++++++++------
 clang/lib/Driver/ToolChains/BareMetal.h            |  19 +-
 .../aarch64-none-elf/include/c++/8.2.1/.keep       |   0
 .../aarch64-none-elf/lib/.keep                     |   0
 .../aarch64-none-elf/lib/crt0.o                    |   0
 .../basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld |   1 +
 .../lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o      |   0
 .../lib/gcc/aarch64-none-elf/8.2.1/crtend.o        |   0
 .../aarch64-none-elf/lib/crt0.o                    |   0
 .../aarch64-none-elf/lib/crtbegin.o                |   0
 .../aarch64-none-elf/lib/crtend.o                  |   0
 .../bin/aarch64-none-elf-ld                        |   1 +
 .../armv6m-none-eabi/include/c++/8.2.1/.keep       |   0
 .../basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep  |   0
 .../basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o |   0
 .../basic_arm_gcc_tree/bin/armv6m-none-eabi-ld     |   1 +
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o      |   0
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtend.o        |   0
 .../armv6m-none-eabi/lib/crt0.o                    |   0
 .../armv6m-none-eabi/lib/crtbegin.o                |   0
 .../armv6m-none-eabi/lib/crtend.o                  |   0
 .../basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld   |   1 +
 clang/test/Driver/aarch64-gnutools.c               |   4 +
 clang/test/Driver/aarch64-toolchain-extra.c        |  28 +++
 clang/test/Driver/aarch64-toolchain.c              |  61 ++++++
 clang/test/Driver/arm-gnutools.c                   |   6 +
 clang/test/Driver/arm-toolchain-extra.c            |  29 +++
 clang/test/Driver/arm-toolchain.c                  |  62 ++++++
 clang/test/Driver/baremetal.cpp                    |  16 ++
 clang/test/Driver/check-no-multlib-warning.c       |  10 +
 32 files changed, 418 insertions(+), 64 deletions(-)
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
 create mode 100755 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
 create mode 100755 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
 create mode 100755 clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
 create mode 100755 clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
 create mode 100644 clang/test/Driver/aarch64-gnutools.c
 create mode 100644 clang/test/Driver/aarch64-toolchain-extra.c
 create mode 100644 clang/test/Driver/aarch64-toolchain.c
 create mode 100644 clang/test/Driver/arm-gnutools.c
 create mode 100644 clang/test/Driver/arm-toolchain-extra.c
 create mode 100644 clang/test/Driver/arm-toolchain.c
 create mode 100644 clang/test/Driver/check-no-multlib-warning.c

diff --git a/clang/docs/Toolchain.rst b/clang/docs/Toolchain.rst
index 958199eb7a2e..d56b21d74c7e 100644
--- a/clang/docs/Toolchain.rst
+++ b/clang/docs/Toolchain.rst
@@ -347,3 +347,8 @@ workarounds for issues discovered in libstdc++, and these are removed
 as fixed libstdc++ becomes sufficiently old.
 
 You can instruct Clang to use libstdc++ with the ``-stdlib=libstdc++`` flag.
+
+GCC Installation
+=================
+Users can point to their GCC installation by using the ``-gcc-toolchain`` or by
+using ``-gcc-install-dir`` flag.
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 29f6480ba935..94224e103875 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -847,6 +847,9 @@ def note_drv_available_multilibs : Note<
   "available multilibs are:%0">;
 def err_drv_multilib_custom_error : Error<
   "multilib configuration error: %0">;
+def warn_drv_multilib_not_available_for_target: Warning<
+  "no multilib structure encoded for Arm, Aarch64 and PPC targets">,
+  InGroup<DiagGroup<"multilib-not-found">>;
 
 def err_drv_experimental_crel : Error<
   "-Wa,--allow-experimental-crel must be specified to use -Wa,--crel. "
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index d8168ed15feb..0fbfe6c77f34 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -31,6 +31,40 @@ using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 
+/// Is the triple {aarch64.aarch64_be}-none-elf?
+static bool isAArch64BareMetal(const llvm::Triple &Triple) {
+  if (Triple.getArch() != llvm::Triple::aarch64 &&
+      Triple.getArch() != llvm::Triple::aarch64_be)
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+static bool isRISCVBareMetal(const llvm::Triple &Triple) {
+  if (!Triple.isRISCV())
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
+}
+
+/// Is the triple powerpc[64][le]-*-none-eabi?
+static bool isPPCBareMetal(const llvm::Triple &Triple) {
+  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
+         Triple.getEnvironment() == llvm::Triple::EABI;
+}
+
 static bool findRISCVMultilibs(const Driver &D,
                                const llvm::Triple &TargetTriple,
                                const ArgList &Args, DetectedMultilibs &Result) {
@@ -95,7 +129,8 @@ static bool findRISCVMultilibs(const Driver &D,
   return false;
 }
 
-static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
+static std::string computeClangRuntimesSysRoot(const Driver &D,
+                                               bool IncludeTriple) {
   if (!D.SysRoot.empty())
     return D.SysRoot;
 
@@ -108,56 +143,123 @@ static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
   return std::string(SysRootDir);
 }
 
-BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
-                     const ArgList &Args)
-    : ToolChain(D, Triple, Args),
-      SysRoot(computeBaseSysRoot(D, /*IncludeTriple=*/true)) {
-  getProgramPaths().push_back(getDriver().Dir);
-
-  findMultilibs(D, Triple, Args);
-  SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
-    for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
-      llvm::sys::path::append(Dir, M.osSuffix(), "lib");
-      getFilePaths().push_back(std::string(Dir));
-      getLibraryPaths().push_back(std::string(Dir));
-    }
+// Only consider the GCC toolchain based on the values provided through the
+// `--gcc-toolchain` and `--gcc-install-dir` flags. The function below returns
+// whether the GCC toolchain was initialized successfully.
+bool BareMetal::initGCCInstallation(const llvm::Triple &Triple,
+                                    const llvm::opt::ArgList &Args) {
+  if (Args.getLastArg(options::OPT_gcc_toolchain) ||
+      Args.getLastArg(clang::driver::options::OPT_gcc_install_dir_EQ)) {
+    GCCInstallation.init(Triple, Args);
+    return GCCInstallation.isValid();
   }
+  return false;
 }
 
-/// Is the triple {aarch64.aarch64_be}-none-elf?
-static bool isAArch64BareMetal(const llvm::Triple &Triple) {
-  if (Triple.getArch() != llvm::Triple::aarch64 &&
-      Triple.getArch() != llvm::Triple::aarch64_be)
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
-
-  return Triple.getEnvironmentName() == "elf";
+// This logic is adapted from RISCVToolChain.cpp as part of the ongoing effort
+// to merge RISCVToolChain into the Baremetal toolchain. It infers the presence
+// of a valid GCC toolchain by checking whether the `crt0.o` file exists in the
+// `bin/../<target-triple>/lib` directory.
+static bool detectGCCToolchainAdjacent(const Driver &D) {
+  SmallString<128> GCCDir;
+  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
+                          "lib/crt0.o");
+  return llvm::sys::fs::exists(GCCDir);
 }
 
-static bool isRISCVBareMetal(const llvm::Triple &Triple) {
-  if (!Triple.isRISCV())
-    return false;
+// If no sysroot is provided the driver will first attempt to infer it from the
+// values of `--gcc-install-dir` or `--gcc-toolchain`, which specify the
+// location of a GCC toolchain.
+// If neither flag is used, the sysroot defaults to either:
+//    - `bin/../<target-triple>`
+//    - `bin/../lib/clang-runtimes/<target-triple>`
+//
+// To use the `clang-runtimes` path, ensure that `../<target-triple>/lib/crt0.o`
+// does not exist relative to the driver.
+std::string BareMetal::computeSysRoot() const {
+  // Use Baremetal::sysroot if it has already been set.
+  if (!SysRoot.empty())
+    return SysRoot;
+
+  // Use the sysroot specified via the `--sysroot` command-line flag, if
+  // provided.
+  const Driver &D = getDriver();
+  if (!D.SysRoot.empty())
+    return D.SysRoot;
 
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
+  // Attempt to infer sysroot from a valid GCC installation.
+  // If no valid GCC installation, check for a GCC toolchain alongside Clang.
+  SmallString<128> inferredSysRoot;
+  if (IsGCCInstallationValid) {
+    llvm::sys::path::append(inferredSysRoot, GCCInstallation.getParentLibPath(),
+                            "..", GCCInstallation.getTriple().str());
+  } else if (detectGCCToolchainAdjacent(D)) {
+    // Use the triple as provided to the driver. Unlike the parsed triple
+    // this has not been normalized to always contain every field.
+    llvm::sys::path::append(inferredSysRoot, D.Dir, "..", D.getTargetTriple());
+  }
+  // If a valid sysroot was inferred and exists, use it
+  if (!inferredSysRoot.empty() && llvm::sys::fs::exists(inferredSysRoot))
+    return std::string(inferredSysRoot);
 
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
+  // Use the clang-runtimes path.
+  return computeClangRuntimesSysRoot(D, /*IncludeTriple*/ true);
+}
 
-  return Triple.getEnvironmentName() == "elf";
+static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
+                                  const Multilib &Multilib,
+                                  StringRef InstallPath,
+                                  ToolChain::path_list &Paths) {
+  if (const auto &PathsCallback = Multilibs.filePathsCallback())
+    for (const auto &Path : PathsCallback(Multilib))
+      addPathIfExists(D, InstallPath + Path, Paths);
 }
 
-/// Is the triple powerpc[64][le]-*-none-eabi?
-static bool isPPCBareMetal(const llvm::Triple &Triple) {
-  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
-         Triple.getEnvironment() == llvm::Triple::EABI;
+// GCC mutltilibs will only work for those targets that have their multlib
+// structure encoded into GCCInstallation. Baremetal toolchain supports ARM,
+// AArch64, RISCV and PPC and of these only RISCV have GCC multilibs hardcoded
+// in GCCInstallation.
+BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
+                     const ArgList &Args)
+    : Generic_ELF(D, Triple, Args) {
+  IsGCCInstallationValid = initGCCInstallation(Triple, Args);
+  std::string ComputedSysRoot = computeSysRoot();
+  if (IsGCCInstallationValid) {
+    if (!isRISCVBareMetal(Triple))
+      D.Diag(clang::diag::warn_drv_multilib_not_available_for_target);
+
+    Multilibs = GCCInstallation.getMultilibs();
+    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
+
+    path_list &Paths = getFilePaths();
+    // Add toolchain/multilib specific file paths.
+    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
+                          GCCInstallation.getInstallPath(), Paths);
+    // Adding filepath for locating crt{begin,end}.o files.
+    Paths.push_back(GCCInstallation.getInstallPath().str());
+    // Adding filepath for locating crt0.o file.
+    Paths.push_back(ComputedSysRoot + "/lib");
+
+    ToolChain::path_list &PPaths = getProgramPaths();
+    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
+    // directory off of the parent of the GCC installation.
+    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
+                           GCCInstallation.getTriple().str() + "/bin")
+                         .str());
+    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
+  } else {
+    getProgramPaths().push_back(getDriver().Dir);
+    findMultilibs(D, Triple, Args);
+    const SmallString<128> SysRootDir(computeSysRoot());
+    if (!SysRootDir.empty()) {
+      for (const Multilib &M : getOrderedMultilibs()) {
+        SmallString<128> Dir(SysRootDir);
+        llvm::sys::path::append(Dir, M.osSuffix(), "lib");
+        getFilePaths().push_back(std::string(Dir));
+        getLibraryPaths().push_back(std::string(Dir));
+      }
+    }
+  }
 }
 
 static void
@@ -216,7 +318,7 @@ getMultilibConfigPath(const Driver &D, const llvm::Triple &Triple,
       return {};
     }
   } else {
-    MultilibPath = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    MultilibPath = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     llvm::sys::path::append(MultilibPath, MultilibFilename);
   }
   return MultilibPath;
@@ -234,7 +336,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
   if (D.getVFS().exists(*MultilibPath)) {
     // If multilib.yaml is found, update sysroot so it doesn't use a target
     // specific suffix
-    SysRoot = computeBaseSysRoot(D, /*IncludeTriple=*/false);
+    SysRoot = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
     SmallVector<StringRef> CustomFlagMacroDefines;
     findMultilibsFromYAML(*this, D, *MultilibPath, Args, Result,
                           CustomFlagMacroDefines);
@@ -242,7 +344,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
     Multilibs = Result.Multilibs;
     MultilibMacroDefines.append(CustomFlagMacroDefines.begin(),
                                 CustomFlagMacroDefines.end());
-  } else if (isRISCVBareMetal(Triple)) {
+  } else if (isRISCVBareMetal(Triple) && !detectGCCToolchainAdjacent(D)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
@@ -263,8 +365,6 @@ Tool *BareMetal::buildStaticLibTool() const {
   return new tools::baremetal::StaticLibTool(*this);
 }
 
-std::string BareMetal::computeSysRoot() const { return SysRoot; }
-
 BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   // Get multilibs in reverse order because they're ordered most-specific last.
   if (!SelectedMultilibs.empty())
@@ -292,10 +392,10 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (std::optional<std::string> Path = getStdlibIncludePath())
     addSystemInclude(DriverArgs, CC1Args, *Path);
 
-  const SmallString<128> SysRoot(computeSysRoot());
-  if (!SysRoot.empty()) {
+  const SmallString<128> SysRootDir(computeSysRoot());
+  if (!SysRootDir.empty()) {
     for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRoot);
+      SmallString<128> Dir(SysRootDir);
       llvm::sys::path::append(Dir, M.includeSuffix());
       llvm::sys::path::append(Dir, "include");
       addSystemInclude(DriverArgs, CC1Args, Dir.str());
@@ -309,6 +409,19 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
   CC1Args.push_back("-nostdsysteminc");
 }
 
+void BareMetal::addLibStdCxxIncludePaths(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args) const {
+  if (!IsGCCInstallationValid)
+    return;
+  const GCCVersion &Version = GCCInstallation.getVersion();
+  StringRef TripleStr = GCCInstallation.getTriple().str();
+  const Multilib &Multilib = GCCInstallation.getMultilib();
+  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
+                           TripleStr, Multilib.includeSuffix(), DriverArgs,
+                           CC1Args);
+}
+
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
@@ -339,23 +452,23 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   };
 
   switch (GetCXXStdlibType(DriverArgs)) {
-    case ToolChain::CST_Libcxx: {
-      SmallString<128> P(D.Dir);
-      llvm::sys::path::append(P, "..", "include");
-      AddCXXIncludePath(P);
-      break;
-    }
-    case ToolChain::CST_Libstdcxx:
-      // We only support libc++ toolchain installation.
-      break;
+  case ToolChain::CST_Libcxx: {
+    SmallString<128> P(D.Dir);
+    llvm::sys::path::append(P, "..", "include");
+    AddCXXIncludePath(P);
+    break;
+  }
+  case ToolChain::CST_Libstdcxx:
+    addLibStdCxxIncludePaths(DriverArgs, CC1Args);
+    break;
   }
 
-  std::string SysRoot(computeSysRoot());
-  if (SysRoot.empty())
+  std::string SysRootDir(computeSysRoot());
+  if (SysRootDir.empty())
     return;
 
   for (const Multilib &M : getOrderedMultilibs()) {
-    SmallString<128> Dir(SysRoot);
+    SmallString<128> Dir(SysRootDir);
     llvm::sys::path::append(Dir, M.gccSuffix());
     switch (GetCXXStdlibType(DriverArgs)) {
     case ToolChain::CST_Libcxx: {
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index f6295bda0a6a..930f8584e643 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -9,6 +9,7 @@
 #ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 
+#include "ToolChains/Gnu.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 
@@ -19,7 +20,7 @@ namespace driver {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
+class LLVM_LIBRARY_VISIBILITY BareMetal : public Generic_ELF {
 public:
   BareMetal(const Driver &D, const llvm::Triple &Triple,
             const llvm::opt::ArgList &Args);
@@ -35,7 +36,8 @@ protected:
   Tool *buildStaticLibTool() const override;
 
 public:
-  bool useIntegratedAs() const override { return true; }
+  bool initGCCInstallation(const llvm::Triple &Triple,
+                           const llvm::opt::ArgList &Args);
   bool isBareMetal() const override { return true; }
   bool isCrossCompiling() const override { return true; }
   bool HasNativeLLVMSupport() const override { return true; }
@@ -48,9 +50,15 @@ public:
 
   StringRef getOSLibName() const override { return "baremetal"; }
 
+  UnwindTableLevel
+  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override {
+    return UnwindTableLevel::None;
+  }
+
   RuntimeLibType GetDefaultRuntimeLibType() const override {
     return ToolChain::RLT_CompilerRT;
   }
+
   CXXStdlibType GetDefaultCXXStdlibType() const override {
     return ToolChain::CST_Libcxx;
   }
@@ -67,6 +75,9 @@ public:
   void AddClangCXXStdlibIncludeArgs(
       const llvm::opt::ArgList &DriverArgs,
       llvm::opt::ArgStringList &CC1Args) const override;
+  void
+  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
+                           llvm::opt::ArgStringList &CC1Args) const override;
   std::string computeSysRoot() const override;
   SanitizerMask getSupportedSanitizers() const override;
 
@@ -80,6 +91,8 @@ private:
 
   std::string SysRoot;
 
+  bool IsGCCInstallationValid;
+
   SmallVector<std::string> MultilibMacroDefines;
 };
 
@@ -104,7 +117,7 @@ public:
 
 class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
 public:
-  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "ld.lld", TC) {}
+  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "linker", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
new file mode 100755
index 000000000000..b23e55619b2f
--- /dev/null
+++ b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
@@ -0,0 +1 @@
+#!/bin/true
diff --git a/clang/test/Driver/aarch64-gnutools.c b/clang/test/Driver/aarch64-gnutools.c
new file mode 100644
index 000000000000..0214639ed380
--- /dev/null
+++ b/clang/test/Driver/aarch64-gnutools.c
@@ -0,0 +1,4 @@
+// RUN: %clang --target=aarch64-none-elf  --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/aarch64-toolchain-extra.c b/clang/test/Driver/aarch64-toolchain-extra.c
new file mode 100644
index 000000000000..2610e962bd69
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain-extra.c
@@ -0,0 +1,28 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in aarch64-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/aarch64-nogcc/bin
+// RUN: ln -s %clang %t/aarch64-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf %t/aarch64-nogcc/aarch64-none-elf
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/aarch64-nogcc/invalid \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/aarch64-nogcc/bin/../aarch64-none-elf \
+// RUN:    --target=aarch64-none-elf --rtlib=libgcc -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/include"
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
new file mode 100644
index 000000000000..7f2c01d928e4
--- /dev/null
+++ b/clang/test/Driver/aarch64-toolchain.c
@@ -0,0 +1,61 @@
+// UNSUPPORTED: system-windows
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL %s
+
+// C-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// C-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "aarch64-unknown-none-elf"
+// C-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL %s
+
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/backward"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/v1"
+// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
diff --git a/clang/test/Driver/arm-gnutools.c b/clang/test/Driver/arm-gnutools.c
new file mode 100644
index 000000000000..6e107f19dabc
--- /dev/null
+++ b/clang/test/Driver/arm-gnutools.c
@@ -0,0 +1,6 @@
+// check that gnu assembler is invoked with arm baremetal as well
+
+// RUN: %clang --target=armv6m-none-eabi  --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -fno-integrated-as %s -### -c \
+// RUN: 2>&1 | FileCheck %s
+
+// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/arm-toolchain-extra.c b/clang/test/Driver/arm-toolchain-extra.c
new file mode 100644
index 000000000000..114de0a8154a
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain-extra.c
@@ -0,0 +1,29 @@
+// A basic clang -cc1 command-line, and simple environment check.
+
+// The tests here are similar to those in arm-toolchain.c, however
+// these tests need to create symlinks to test directory trees in order to
+// set up the environment and therefore shell support is required.
+// REQUIRES: shell
+// UNSUPPORTED: system-windows
+
+// If there is no GCC install detected then the driver searches for executables
+// and runtime starting from the directory tree above the driver itself.
+// The test below checks that the driver correctly finds the linker and
+// runtime if and only if they exist.
+//
+// RUN: rm -rf %t
+// RUN: mkdir -p %t/arm-nogcc/bin
+// RUN: ln -s %clang %t/arm-nogcc/bin/clang
+// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi %t/arm-nogcc/armv6m-none-eabi
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --gcc-toolchain=%t/arm-nogcc/invalid \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
+// RUN:    --sysroot=%t/arm-nogcc/bin/../armv6m-none-eabi \
+// RUN:    --target=armv6m-none-eabi --rtlib=libgcc -fuse-ld=ld 2>&1 \
+// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
+
+// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/include"
+
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
new file mode 100644
index 000000000000..2e38461fb7a3
--- /dev/null
+++ b/clang/test/Driver/arm-toolchain.c
@@ -0,0 +1,62 @@
+// UNSUPPORTED: system-windows
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL %s
+
+// C-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// C-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+
+// RUN: %clang -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL-NOSYSROOT %s
+
+// C-ARM-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
+// C-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL %s
+
+// CXX-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1" 
+// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/backward"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1"
+// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
+
+// RUN: %clangxx -### %s -fuse-ld= \
+// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc \
+// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
+// RUN:   --sysroot=  2>&1 \
+// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX %s
+
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/v1"
+// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index a80aa9b43711..2ac83402dda3 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -196,6 +196,22 @@
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv32-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV32-NO-HOST-INC %s
+// CHECK-RISCV32-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV32-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
+// RUN: %clang -no-canonical-prefixes %s -### --target=riscv64-unknown-elf 2>&1 \
+// RUN:   | FileCheck --check-prefix=CHECK-RISCV64-NO-HOST-INC %s
+// CHECK-RISCV64-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
+// CHECK-RISCV64-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
+// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
+
 // RUN: %clang %s -### --target=riscv64-unknown-elf -o %t.out -L some/directory/user/asked/for \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/Driver/check-no-multlib-warning.c b/clang/test/Driver/check-no-multlib-warning.c
new file mode 100644
index 000000000000..9a0d7cee450a
--- /dev/null
+++ b/clang/test/Driver/check-no-multlib-warning.c
@@ -0,0 +1,10 @@
+// UNSUPPORTED: system-windows
+
+
+// RUN: %clang --target=armv6m-none-eabi --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=aarch64-none-elf --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -### 2>&1 | FileCheck %s
+// RUN: %clang --target=riscv32-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv32_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+// RUN: %clang --target=riscv64-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv64_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
+
+// CHECK: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
+// NOCHECK-NOT: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
-- 
cgit v1.2.3


From e07b1b26c38ba48af247b370a29eeb9879cefc97 Mon Sep 17 00:00:00 2001
From: Rajveer Singh Bharadwaj <rajveer.developer@icloud.com>
Date: Wed, 18 Jun 2025 12:59:27 +0530
Subject: [DAG] Implement SDPatternMatch `m_Abs()` matcher (#144512)

---
 llvm/include/llvm/CodeGen/SDPatternMatch.h              |  4 ++++
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp           | 12 +++---------
 llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp |  4 ++++
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index 2e3807a2dfff..d413227c4d96 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -938,6 +938,10 @@ template <typename Opnd> inline UnaryOpc_match<Opnd> m_Trunc(const Opnd &Op) {
   return UnaryOpc_match<Opnd>(ISD::TRUNCATE, Op);
 }
 
+template <typename Opnd> inline UnaryOpc_match<Opnd> m_Abs(const Opnd &Op) {
+  return UnaryOpc_match<Opnd>(ISD::ABS, Op);
+}
+
 /// Match a zext or identity
 /// Allows to peek through optional extensions
 template <typename Opnd> inline auto m_ZExtOrSelf(const Opnd &Op) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d14615dcbc5e..934199e414c7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -11260,19 +11260,13 @@ SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
   if (N->getOpcode() == ISD::TRUNCATE)
     N = N->getOperand(0).getNode();
 
-  if (N->getOpcode() != ISD::ABS)
-    return SDValue();
-
   EVT VT = N->getValueType(0);
-  SDValue AbsOp1 = N->getOperand(0);
   SDValue Op0, Op1;
 
-  if (AbsOp1.getOpcode() != ISD::SUB)
+  if (!sd_match(N, m_Abs(m_Sub(m_Value(Op0), m_Value(Op1)))))
     return SDValue();
 
-  Op0 = AbsOp1.getOperand(0);
-  Op1 = AbsOp1.getOperand(1);
-
+  SDValue AbsOp0 = N->getOperand(0);
   unsigned Opc0 = Op0.getOpcode();
 
   // Check if the operands of the sub are (zero|sign)-extended.
@@ -11282,7 +11276,7 @@ SDValue DAGCombiner::foldABSToABD(SDNode *N, const SDLoc &DL) {
        Opc0 != ISD::SIGN_EXTEND_INREG)) {
     // fold (abs (sub nsw x, y)) -> abds(x, y)
     // Don't fold this for unsupported types as we lose the NSW handling.
-    if (AbsOp1->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) &&
+    if (AbsOp0->getFlags().hasNoSignedWrap() && hasOperation(ISD::ABDS, VT) &&
         TLI.preferABDSToABSWithNSW(VT)) {
       SDValue ABD = DAG.getNode(ISD::ABDS, DL, VT, Op0, Op1);
       return DAG.getZExtOrTrunc(ABD, DL, SrcVT);
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index 1b590aa33bd8..2162588aadfd 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -388,6 +388,8 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
   SDValue SExt = DAG->getNode(ISD::SIGN_EXTEND, DL, Int64VT, Op0);
   SDValue Trunc = DAG->getNode(ISD::TRUNCATE, DL, Int32VT, Op1);
 
+  SDValue Abs = DAG->getNode(ISD::ABS, DL, Int32VT, Op0);
+
   SDValue Sub = DAG->getNode(ISD::SUB, DL, Int32VT, Trunc, Op0);
   SDValue Neg = DAG->getNegative(Op0, DL, Int32VT);
   SDValue Not = DAG->getNOT(DL, Op0, Int32VT);
@@ -417,6 +419,8 @@ TEST_F(SelectionDAGPatternMatchTest, matchUnaryOp) {
   EXPECT_FALSE(sd_match(ZExt, m_SExtLike(m_Value())));
   EXPECT_TRUE(sd_match(Trunc, m_Trunc(m_Specific(Op1))));
 
+  EXPECT_TRUE(sd_match(Abs, m_Abs(m_Specific(Op0))));
+
   EXPECT_TRUE(sd_match(Neg, m_Neg(m_Value())));
   EXPECT_TRUE(sd_match(Not, m_Not(m_Value())));
   EXPECT_FALSE(sd_match(ZExt, m_Neg(m_Value())));
-- 
cgit v1.2.3


From a38932ac3c0a16226e3dde7f1532f117959c58df Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler@amd.com>
Date: Wed, 18 Jun 2025 09:49:32 +0200
Subject: Revert "[GlobalISel] prevent G_UNMERGE_VALUES for vectors with
 different elements" (#144650)

Reverts llvm/llvm-project#133335
---
 .../GlobalISel/LegalizationArtifactCombiner.h      |  5 +-
 .../CodeGen/AMDGPU/GlobalISel/insertelement.ll     | 55 ----------------------
 2 files changed, 1 insertion(+), 59 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 8f560c42082f..22f6a5fde546 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -997,7 +997,6 @@ public:
 
       // Recognize UnmergeSrc that can be unmerged to DstTy directly.
       // Types have to be either both vector or both non-vector types.
-      // In case of vector types, the scalar elements need to match.
       // Merge-like opcodes are combined one at the time. First one creates new
       // unmerge, following should use the same unmerge (builder performs CSE).
       //
@@ -1006,9 +1005,7 @@ public:
       // %AnotherDst:_(DstTy) = G_merge_like_opcode %2:_(EltTy), %3
       //
       // %Dst:_(DstTy), %AnotherDst = G_UNMERGE_VALUES %UnmergeSrc
-      if (((!DstTy.isVector() && !UnmergeSrcTy.isVector()) ||
-           (DstTy.isVector() && UnmergeSrcTy.isVector() &&
-            DstTy.getScalarType() == UnmergeSrcTy.getScalarType())) &&
+      if ((DstTy.isVector() == UnmergeSrcTy.isVector()) &&
           (Elt0UnmergeIdx % NumMIElts == 0) &&
           getCoverTy(UnmergeSrcTy, DstTy) == UnmergeSrcTy) {
         if (!isSequenceFromUnmerge(MI, 0, Unmerge, Elt0UnmergeIdx, NumMIElts,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 132a89478c5f..8134eb3ca2af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -6506,58 +6506,3 @@ entry:
   %insert = insertelement <5 x double> %vec, double %val, i32 %idx
   ret <5 x double> %insert
 }
-
-; Found by fuzzer, reduced with llvm-reduce.
-define amdgpu_kernel void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
-; GPRIDX-LABEL: insert_very_small_from_very_large:
-; GPRIDX:       ; %bb.0: ; %bb
-; GPRIDX-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x0
-; GPRIDX-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x40
-; GPRIDX-NEXT:    s_waitcnt lgkmcnt(0)
-; GPRIDX-NEXT:    s_lshr_b32 s2, s12, 1
-; GPRIDX-NEXT:    s_and_b32 s2, s2, 1
-; GPRIDX-NEXT:    s_lshl_b32 s2, s2, 1
-; GPRIDX-NEXT:    v_mov_b32_e32 v0, s0
-; GPRIDX-NEXT:    v_mov_b32_e32 v2, s2
-; GPRIDX-NEXT:    v_mov_b32_e32 v1, s1
-; GPRIDX-NEXT:    flat_store_byte v[0:1], v2
-; GPRIDX-NEXT:    s_endpgm
-;
-; GFX10-LABEL: insert_very_small_from_very_large:
-; GFX10:       ; %bb.0: ; %bb
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_load_dwordx16 s[12:27], s[8:9], 0x0
-; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[8:9], 0x40
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_lshr_b32 s2, s12, 1
-; GFX10-NEXT:    v_mov_b32_e32 v0, s0
-; GFX10-NEXT:    s_and_b32 s2, s2, 1
-; GFX10-NEXT:    v_mov_b32_e32 v1, s1
-; GFX10-NEXT:    s_lshl_b32 s2, s2, 1
-; GFX10-NEXT:    v_mov_b32_e32 v2, s2
-; GFX10-NEXT:    flat_store_byte v[0:1], v2
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: insert_very_small_from_very_large:
-; GFX11:       ; %bb.0: ; %bb
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b512 s[8:23], s[4:5], 0x0
-; GFX11-NEXT:    s_load_b64 s[0:1], s[4:5], 0x40
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_lshr_b32 s2, s8, 1
-; GFX11-NEXT:    v_mov_b32_e32 v0, s0
-; GFX11-NEXT:    s_and_b32 s2, s2, 1
-; GFX11-NEXT:    v_mov_b32_e32 v1, s1
-; GFX11-NEXT:    s_lshl_b32 s2, s2, 1
-; GFX11-NEXT:    v_mov_b32_e32 v2, s2
-; GFX11-NEXT:    flat_store_b8 v[0:1], v2
-; GFX11-NEXT:    s_endpgm
-bb:
-  %a = bitcast <32 x i16> %L3 to i512
-  %b = trunc i512 %a to i8
-  %c = trunc i8 %b to i2
-  %d = bitcast i2 %c to <2 x i1>
-  %insert = insertelement <2 x i1> %d, i1 false, i32 0
-  store <2 x i1> %insert, ptr %ptr, align 1
-  ret void
-}
-- 
cgit v1.2.3


From 49df87e71b73b230ecb21335dcb5f5390eebdab3 Mon Sep 17 00:00:00 2001
From: Simon Tatham <simon.tatham@arm.com>
Date: Wed, 18 Jun 2025 08:57:51 +0100
Subject: [libc][printf] Fix out-of-range shift in float320 printf (#144542)

If you enable `LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_FLOAT320` and use a
`%f` style printf format directive to print a nonzero number too small
to show up in the output digits, e.g. `printf("%.2f", 0.001)`, then the
output would be intermittently incorrect, because
`DyadicFloat::as_mantissa_type_rounded` would try to shift the 320-bit
mantissa right by more than 320 bits, invoking the 'undefined behavior'
clause commented in the `shift()` function in `big_int.h`.

There were already tests in the libc test suite exercising this case,
e.g. the subnormal tests in `LlvmLibcSPrintfTest.FloatDecimalConv` use
`%f` at the default precision of 6 decimal places on tiny numbers such
as 2^-1027. But because the behavior is undefined, they don't visibly
fail all the time, and in all previous test runs we'd tried with
USE_FLOAT320, they had got lucky.

The fix is simply to detect an out-of-range right shift before doing it,
and instead just set the output value to zero.
---
 libc/src/__support/FPUtil/dyadic_float.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h
index 6c3e1520e5af..4c77d3c541cd 100644
--- a/libc/src/__support/FPUtil/dyadic_float.h
+++ b/libc/src/__support/FPUtil/dyadic_float.h
@@ -465,7 +465,10 @@ template <size_t Bits> struct DyadicFloat {
         // exponents coming in to this function _shouldn't_ be that large). The
         // result should always end up as a positive size_t.
         size_t shift = -static_cast<size_t>(exponent);
-        new_mant >>= shift;
+        if (shift >= Bits)
+          new_mant = 0;
+        else
+          new_mant >>= shift;
         round_dir = rounding_direction(mantissa, shift, sign);
         if (round_dir > 0)
           ++new_mant;
-- 
cgit v1.2.3


From ba40a7bc2e65be86ac23c9cf6038ac085dda77eb Mon Sep 17 00:00:00 2001
From: Mel Chen <mel.chen@sifive.com>
Date: Wed, 18 Jun 2025 16:03:20 +0800
Subject: [LoopVectorize] Vectorize fixed-order recurrence with vscale x 1.
 (#142772)

When the fixed-order recurrence phi is live-out from the loop, the
vectorizer uses VPInstruction::ExtractPenultimateElement to extract the
penultimate element from the recurrence vector. However, this is not
feasible when the VF is vscale x 1, since vscale could be 1, making the
vector contain only one element.

This patch changes the behavior for vscale x 1 by extracting the last
element from the vector produced by splicing the recurrence phi and the
previous value. This ensures we can still determine the correct live-out
value of the recurrence phi.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    | 22 ++++--
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp     |  3 -
 .../RISCV/first-order-recurrence-scalable-vf1.ll   | 57 ++++++++------
 .../first-order-recurrence-scalable-vf1.ll         | 90 ++++++++++++++++++++--
 4 files changed, 130 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index f887b34e7642..16d48b06dce4 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -6163,11 +6163,6 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I,
 
     // First-order recurrences are replaced by vector shuffles inside the loop.
     if (VF.isVector() && Legal->isFixedOrderRecurrence(Phi)) {
-      // For <vscale x 1 x i64>, if vscale = 1 we are unable to extract the
-      // penultimate value of the recurrence.
-      // TODO: Consider vscale_range info.
-      if (VF.isScalable() && VF.getKnownMinValue() == 1)
-        return InstructionCost::getInvalid();
       SmallVector<int> Mask(VF.getKnownMinValue());
       std::iota(Mask.begin(), Mask.end(), VF.getKnownMinValue() - 1);
       return TTI.getShuffleCost(TargetTransformInfo::SK_Splice,
@@ -8556,13 +8551,17 @@ addUsersInExitBlocks(VPlan &Plan,
 /// users in the original exit block using the VPIRInstruction wrapping to the
 /// LCSSA phi.
 static void addExitUsersForFirstOrderRecurrences(
-    VPlan &Plan, SetVector<VPIRInstruction *> &ExitUsersToFix) {
+    VPlan &Plan, SetVector<VPIRInstruction *> &ExitUsersToFix, VFRange &Range) {
   VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
   auto *ScalarPHVPBB = Plan.getScalarPreheader();
   auto *MiddleVPBB = Plan.getMiddleBlock();
   VPBuilder ScalarPHBuilder(ScalarPHVPBB);
   VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi());
 
+  auto IsScalableOne = [](ElementCount VF) -> bool {
+    return VF == ElementCount::getScalable(1);
+  };
+
   for (auto &HeaderPhi : VectorRegion->getEntryBasicBlock()->phis()) {
     auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&HeaderPhi);
     if (!FOR)
@@ -8644,6 +8643,15 @@ static void addExitUsersForFirstOrderRecurrences(
     for (VPIRInstruction *ExitIRI : ExitUsersToFix) {
       if (ExitIRI->getOperand(0) != FOR)
         continue;
+      // For VF vscale x 1, if vscale = 1, we are unable to extract the
+      // penultimate value of the recurrence. Instead, we rely on function
+      // addUsersInExitBlocks to extract the last element from the result of
+      // VPInstruction::FirstOrderRecurrenceSplice by leaving the user of the
+      // recurrence phi in ExitUsersToFix.
+      // TODO: Consider vscale_range info and UF.
+      if (LoopVectorizationPlanner::getDecisionAndClampRange(IsScalableOne,
+                                                             Range))
+        return;
       VPValue *PenultimateElement = MiddleBuilder.createNaryOp(
           VPInstruction::ExtractPenultimateElement, {FOR->getBackedgeValue()},
           {}, "vector.recur.extract.for.phi");
@@ -8858,7 +8866,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
   addScalarResumePhis(RecipeBuilder, *Plan, IVEndValues);
   SetVector<VPIRInstruction *> ExitUsersToFix =
       collectUsersInLatchExitBlock(*Plan);
-  addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix);
+  addExitUsersForFirstOrderRecurrences(*Plan, ExitUsersToFix, Range);
   addUsersInExitBlocks(*Plan, ExitUsersToFix);
 
   // ---------------------------------------------------------------------------
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 1ed0b97849a8..f3b5c8cfa988 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3680,9 +3680,6 @@ VPFirstOrderRecurrencePHIRecipe::computeCost(ElementCount VF,
   if (VF.isScalar())
     return Ctx.TTI.getCFInstrCost(Instruction::PHI, Ctx.CostKind);
 
-  if (VF == ElementCount::getScalable(1))
-    return InstructionCost::getInvalid();
-
   return 0;
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
index d34098545716..e3f9540ff3df 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/first-order-recurrence-scalable-vf1.ll
@@ -1,56 +1,63 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -p loop-vectorize -S %s | FileCheck %s
+; RUN: opt -p loop-vectorize -scalable-vectorization=on -S %s | FileCheck %s
 
 target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
 target triple = "riscv64-unknown-linux-gnu"
 
-; Make sure we do not pick <vscale x 1 x i64> as VF for a loop with a
-; first-order recurrence.
 define i64 @pr97452_scalable_vf1_for(ptr %src, ptr noalias %dst) #0 {
 ; CHECK-LABEL: define i64 @pr97452_scalable_vf1_for(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 23, [[TMP0]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <4 x i64> [ <i64 poison, i64 poison, i64 poison, i64 0>, %[[VECTOR_PH]] ], [ [[WIDE_LOAD1:%.*]], %[[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 4
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP1]], align 8
-; CHECK-NEXT:    [[WIDE_LOAD1]] = load <4 x i64>, ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <4 x i64> [[VECTOR_RECUR]], <4 x i64> [[WIDE_LOAD]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
-; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <4 x i64> [[WIDE_LOAD]], <4 x i64> [[WIDE_LOAD1]], <4 x i32> <i32 3, i32 4, i32 5, i32 6>
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 1 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <vscale x 1 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 4
-; CHECK-NEXT:    store <4 x i64> [[TMP3]], ptr [[TMP9]], align 8
-; CHECK-NEXT:    store <4 x i64> [[TMP4]], ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16
-; CHECK-NEXT:    br i1 [[TMP6]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP7]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <4 x i64> [[WIDE_LOAD1]], i32 2
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i64> [[WIDE_LOAD1]], i32 3
-; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP11]], 1
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <vscale x 1 x i64> [[TMP7]], i32 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP15:%.*]] = sub i32 [[TMP14]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 1 x i64> [[WIDE_LOAD]], i32 [[TMP15]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 23, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
-; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP]], align 8
+; CHECK-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
+; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP_SRC]], align 8
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 22
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[VECTOR_RECUR_EXTRACT_FOR_PHI]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[TMP13]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
index 98a942a50107..b20d59bd5760 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-scalable-vf1.ll
@@ -8,17 +8,51 @@ define i64 @pr97452_scalable_vf1_for_live_out(ptr %src) {
 ; CHECK-LABEL: define i64 @pr97452_scalable_vf1_for_live_out(
 ; CHECK-SAME: ptr [[SRC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 23, [[TMP0]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 1 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <vscale x 1 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP10:%.*]] = sub i32 [[TMP9]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <vscale x 1 x i64> [[TMP7]], i32 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 1 x i64> [[WIDE_LOAD]], i32 [[TMP13]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 23, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[L:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 22
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ]
+; CHECK-NEXT:    [[RES:%.*]] = phi i64 [ [[FOR]], %[[LOOP]] ], [ [[TMP11]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i64 [[RES]]
 ;
 entry:
@@ -43,17 +77,51 @@ define void @pr97452_scalable_vf1_for_no_live_out(ptr %src, ptr noalias %dst) {
 ; CHECK-LABEL: define void @pr97452_scalable_vf1_for_no_live_out(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 23, [[TMP0]]
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 23, [[TMP1]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 23, [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP4:%.*]] = sub i32 [[TMP3]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 1 x i64> poison, i64 0, i32 [[TMP4]]
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VECTOR_RECUR:%.*]] = phi <vscale x 1 x i64> [ [[VECTOR_RECUR_INIT]], %[[VECTOR_PH]] ], [ [[WIDE_LOAD:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD]] = load <vscale x 1 x i64>, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = call <vscale x 1 x i64> @llvm.vector.splice.nxv1i64(<vscale x 1 x i64> [[VECTOR_RECUR]], <vscale x 1 x i64> [[WIDE_LOAD]], i32 -1)
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
+; CHECK-NEXT:    store <vscale x 1 x i64> [[TMP7]], ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
+; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP11]], 1
+; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 1 x i64> [[WIDE_LOAD]], i32 [[TMP12]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i64 23, [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i64 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[L:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[FOR:%.*]] = phi i64 [ [[SCALAR_RECUR_INIT]], %[[SCALAR_PH]] ], [ [[L:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr inbounds i64, ptr [[SRC]], i64 [[IV]]
 ; CHECK-NEXT:    [[L]] = load i64, ptr [[GEP]], align 8
 ; CHECK-NEXT:    [[GEP_DST:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[IV]]
 ; CHECK-NEXT:    store i64 [[FOR]], ptr [[GEP_DST]], align 8
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV]], 22
-; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]]
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
 ; CHECK:       [[EXIT]]:
 ; CHECK-NEXT:    ret void
 ;
@@ -74,3 +142,11 @@ loop:
 exit:
   ret void
 }
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]}
+; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]}
+;.
-- 
cgit v1.2.3


From ca29c632f06fc0e02ebbbb9fbdc73e3abd6b096b Mon Sep 17 00:00:00 2001
From: Pengcheng Wang <wangpengcheng.pp@bytedance.com>
Date: Wed, 18 Jun 2025 16:11:18 +0800
Subject: [RISCV] Support non-power-of-2 types when expanding memcmp

We can convert non-power-of-2 types into extended value types
and then they will be widen.

Reviewers: lukel97

Reviewed By: lukel97

Pull Request: https://github.com/llvm/llvm-project/pull/114971
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp        |  23 +-
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp |  21 +-
 llvm/test/CodeGen/RISCV/icmp-non-byte-sized.ll     |  41 ++
 llvm/test/CodeGen/RISCV/memcmp-optsize.ll          | 800 +++++++++------------
 llvm/test/CodeGen/RISCV/memcmp.ll                  | 800 +++++++++------------
 5 files changed, 709 insertions(+), 976 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/icmp-non-byte-sized.ll

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e670567bd184..b8ef221742a2 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -16190,10 +16190,6 @@ combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
     return SDValue();
 
   unsigned OpSize = OpVT.getSizeInBits();
-  // TODO: Support non-power-of-2 types.
-  if (!isPowerOf2_32(OpSize))
-    return SDValue();
-
   // The size should be larger than XLen and smaller than the maximum vector
   // size.
   if (OpSize <= Subtarget.getXLen() ||
@@ -16214,14 +16210,25 @@ combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
           Attribute::NoImplicitFloat))
     return SDValue();
 
+  // Bail out for non-byte-sized types.
+  if (!OpVT.isByteSized())
+    return SDValue();
+
   unsigned VecSize = OpSize / 8;
-  EVT VecVT = MVT::getVectorVT(MVT::i8, VecSize);
-  EVT CmpVT = MVT::getVectorVT(MVT::i1, VecSize);
+  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, VecSize);
+  EVT CmpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, VecSize);
 
   SDValue VecX = DAG.getBitcast(VecVT, X);
   SDValue VecY = DAG.getBitcast(VecVT, Y);
-  SDValue Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
-  return DAG.getSetCC(DL, VT, DAG.getNode(ISD::VECREDUCE_OR, DL, XLenVT, Cmp),
+  SDValue Mask = DAG.getAllOnesConstant(DL, CmpVT);
+  SDValue VL = DAG.getConstant(VecSize, DL, XLenVT);
+
+  SDValue Cmp = DAG.getNode(ISD::VP_SETCC, DL, CmpVT, VecX, VecY,
+                            DAG.getCondCode(ISD::SETNE), Mask, VL);
+  return DAG.getSetCC(DL, VT,
+                      DAG.getNode(ISD::VP_REDUCE_OR, DL, XLenVT,
+                                  DAG.getConstant(0, DL, XLenVT), Cmp, Mask,
+                                  VL),
                       DAG.getConstant(0, DL, XLenVT), CC);
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index aadda2ce8552..46e30ce4c18a 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -2985,20 +2985,13 @@ RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
   }
 
   if (IsZeroCmp && ST->hasVInstructions()) {
-    unsigned RealMinVLen = ST->getRealMinVLen();
-    // Support Fractional LMULs if the lengths are larger than XLen.
-    // TODO: Support non-power-of-2 types.
-    for (unsigned FLMUL = 8; FLMUL >= 2; FLMUL /= 2) {
-      unsigned Len = RealMinVLen / FLMUL;
-      if (Len > ST->getXLen())
-        Options.LoadSizes.insert(Options.LoadSizes.begin(), Len / 8);
-    }
-    for (unsigned LMUL = 1; LMUL <= ST->getMaxLMULForFixedLengthVectors();
-         LMUL *= 2) {
-      unsigned Len = RealMinVLen * LMUL;
-      if (Len > ST->getXLen())
-        Options.LoadSizes.insert(Options.LoadSizes.begin(), Len / 8);
-    }
+    unsigned VLenB = ST->getRealMinVLen() / 8;
+    // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
+    // `VLenB * MaxLMUL` so that it fits in a single register group.
+    unsigned MinSize = ST->getXLen() / 8 + 1;
+    unsigned MaxSize = VLenB * ST->getMaxLMULForFixedLengthVectors();
+    for (unsigned Size = MinSize; Size <= MaxSize; Size++)
+      Options.LoadSizes.insert(Options.LoadSizes.begin(), Size);
   }
   return Options;
 }
diff --git a/llvm/test/CodeGen/RISCV/icmp-non-byte-sized.ll b/llvm/test/CodeGen/RISCV/icmp-non-byte-sized.ll
new file mode 100644
index 000000000000..fca6238548aa
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/icmp-non-byte-sized.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=riscv32 -mattr=+v -O2 < %s | FileCheck %s --check-prefix=CHECK-RV32
+; RUN: llc -mtriple=riscv64 -mattr=+v -O2 < %s | FileCheck %s --check-prefix=CHECK-RV64
+
+define i1 @icmp_non_byte_type(ptr %p1, ptr %p2) nounwind {
+; CHECK-RV32-LABEL: icmp_non_byte_type:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-RV32-NEXT:    lw a3, 4(a0)
+; CHECK-RV32-NEXT:    lw a4, 8(a0)
+; CHECK-RV32-NEXT:    lw a0, 12(a0)
+; CHECK-RV32-NEXT:    lw a5, 12(a1)
+; CHECK-RV32-NEXT:    lw a6, 4(a1)
+; CHECK-RV32-NEXT:    lw a7, 8(a1)
+; CHECK-RV32-NEXT:    lw a1, 0(a1)
+; CHECK-RV32-NEXT:    xor a0, a0, a5
+; CHECK-RV32-NEXT:    xor a3, a3, a6
+; CHECK-RV32-NEXT:    xor a4, a4, a7
+; CHECK-RV32-NEXT:    xor a1, a2, a1
+; CHECK-RV32-NEXT:    or a0, a3, a0
+; CHECK-RV32-NEXT:    or a1, a1, a4
+; CHECK-RV32-NEXT:    or a0, a1, a0
+; CHECK-RV32-NEXT:    seqz a0, a0
+; CHECK-RV32-NEXT:    ret
+;
+; CHECK-RV64-LABEL: icmp_non_byte_type:
+; CHECK-RV64:       # %bb.0:
+; CHECK-RV64-NEXT:    ld a2, 0(a0)
+; CHECK-RV64-NEXT:    ld a0, 8(a0)
+; CHECK-RV64-NEXT:    ld a3, 8(a1)
+; CHECK-RV64-NEXT:    ld a1, 0(a1)
+; CHECK-RV64-NEXT:    xor a0, a0, a3
+; CHECK-RV64-NEXT:    xor a1, a2, a1
+; CHECK-RV64-NEXT:    or a0, a1, a0
+; CHECK-RV64-NEXT:    seqz a0, a0
+; CHECK-RV64-NEXT:    ret
+  %v1 = load i127, ptr %p1
+  %v2 = load i127, ptr %p2
+  %ret = icmp eq i127 %v1, %v2
+  ret i1 %ret
+}
diff --git a/llvm/test/CodeGen/RISCV/memcmp-optsize.ll b/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
index 3742383675b9..0d57e4201512 100644
--- a/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
+++ b/llvm/test/CodeGen/RISCV/memcmp-optsize.ll
@@ -517,17 +517,99 @@ define i32 @bcmp_size_5(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_5:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lbu a0, 4(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lbu a1, 4(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 5, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 5)
   ret i32 %bcmp
@@ -614,17 +696,99 @@ define i32 @bcmp_size_6(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_6:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lhu a0, 4(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lhu a1, 4(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 6)
   ret i32 %bcmp
@@ -711,17 +875,99 @@ define i32 @bcmp_size_7(ptr %s1, ptr %s2) nounwind optsize {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_7:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lw a0, 3(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lw a1, 3(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 7, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 7)
   ret i32 %bcmp
@@ -1069,33 +1315,21 @@ define i32 @bcmp_size_15(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_15:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 7(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 11(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 7(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 11(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a3, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 15, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_15:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 7(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 7(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 15, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1477,57 +1711,21 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 15(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 19(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 23(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 27(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 15(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 19(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 23(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 27(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, t3, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, t2, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t4, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a2, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a5, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a1, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 15(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 23(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 15(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 23(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a3, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1875,129 +2073,23 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -48
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 8(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 4(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 24(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 28(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 24(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 28(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 47(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 51(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 55(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 59(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 31(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 35(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 39(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 43(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, t1, s2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, a7, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 31(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 35(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 39(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 43(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t0, s1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 47(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 51(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 55(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 59(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, s6, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, s8, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, s4, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, s9, s2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s5, s1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, s7, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, s3, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, a6, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t0, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, a7, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t1, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a7, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t0, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a6, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v16
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 4(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 48
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 31(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 39(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 47(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 55(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 31(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 39(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 47(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 55(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, t3, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, t2, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t4, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a2, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a5, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a1, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v16
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -2315,270 +2407,24 @@ define i32 @bcmp_size_127(ptr %s1, ptr %s2) nounwind optsize {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_127:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -96
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 32(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 36(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 40(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 44(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 48(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 52(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 56(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 60(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 24(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 28(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 60(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 24(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 28(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 36(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 40(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 44(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a3, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a4, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 32(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 56(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 48(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a6, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 28(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a7, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 24(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t0, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 20(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t1, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 16(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 107(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t5, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 75(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, t6, s8
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 123(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s2, s2, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s0, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, t4, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 83(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 87(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 91(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s1, s1, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 107(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, t3, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 91(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, t2, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 123(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, a5, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 75(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s5, s11, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s7, s8, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 87(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 83(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s3, s10, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 115(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s6, ra, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 115(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s4, s4, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 119(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 119(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s10, s10, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 71(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 67(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 67(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 71(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 99(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 99(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, a5, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 103(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 103(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, a4, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, s9, s8
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, s11, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, ra, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a5, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 95(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 63(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 111(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 79(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 79(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 111(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 63(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 95(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s9, s9, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, s8, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t2, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t3, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t4, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, s1, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t5, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, s0, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, s2, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, t6, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 12(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t2, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t1, t2, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t2, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t3, t3, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 28(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t4, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 32(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t5, t5, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, t6, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or s0, s0, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, s0, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t5, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t3, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t1, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a6, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t4, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t0, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v24
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 96
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_127:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, -96
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 32(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 40(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 48(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 56(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 32(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 40(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 48(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 56(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t6, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 95(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 103(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 111(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 119(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 63(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 71(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 79(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 87(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t1, t1, s2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, a7, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 63(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 71(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 79(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 87(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t0, s1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 95(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 103(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 111(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 119(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, t6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, t2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, s10
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, s6, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t2, s8, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t4, s4, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t6, s9, s2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor s0, s5, s1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t5, s7, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t3, s3, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a6, a6, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or t0, t0, t6
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, a3, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a7, a7, t2
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t1, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a7, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, t0, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a6, a2
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v24
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 96
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 127)
diff --git a/llvm/test/CodeGen/RISCV/memcmp.ll b/llvm/test/CodeGen/RISCV/memcmp.ll
index f9a6dbba04fc..0caab1f5ce2f 100644
--- a/llvm/test/CodeGen/RISCV/memcmp.ll
+++ b/llvm/test/CodeGen/RISCV/memcmp.ll
@@ -517,17 +517,99 @@ define i32 @bcmp_size_5(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_5:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lbu a0, 4(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lbu a1, 4(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 5, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_5:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lbu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lbu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 5)
   ret i32 %bcmp
@@ -614,17 +696,99 @@ define i32 @bcmp_size_6(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_6:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lhu a0, 4(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lhu a1, 4(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 6, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_6:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lhu a0, 4(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lhu a1, 4(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 6)
   ret i32 %bcmp
@@ -711,17 +875,99 @@ define i32 @bcmp_size_7(ptr %s1, ptr %s2) nounwind {
 ; CHECK-ALIGNED-RV64-V-NEXT:    addi sp, sp, 16
 ; CHECK-ALIGNED-RV64-V-NEXT:    ret
 ;
-; CHECK-UNALIGNED-LABEL: bcmp_size_7:
-; CHECK-UNALIGNED:       # %bb.0: # %entry
-; CHECK-UNALIGNED-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-NEXT:    lw a0, 3(a0)
-; CHECK-UNALIGNED-NEXT:    lw a3, 0(a1)
-; CHECK-UNALIGNED-NEXT:    lw a1, 3(a1)
-; CHECK-UNALIGNED-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-NEXT:    ret
+; CHECK-UNALIGNED-RV32-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-ZBB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-ZBKB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-ZBKB-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-ZBKB:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-ZBKB-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 7, e8, mf2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
+; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    ret
+;
+; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_7:
+; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a2, 0(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a0, 3(a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a3, 0(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    lw a1, 3(a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
+; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
+; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 7)
   ret i32 %bcmp
@@ -1069,33 +1315,21 @@ define i32 @bcmp_size_15(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_15:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 7(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 11(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 7(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 11(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a3, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 15, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_15:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 7(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 7(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 15, e8, m1, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v9, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v8, v8, v9
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v8
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -1555,57 +1789,21 @@ define i32 @bcmp_size_31(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 15(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 19(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 23(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 27(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 15(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 19(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 23(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 27(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, t3, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, t2, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, t4, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a2, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a5, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a3, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a1, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_31:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 15(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 23(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 15(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 23(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a3, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetivli zero, 31, e8, m2, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v10, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v12, v8, v10
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v12
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -2109,129 +2307,23 @@ define i32 @bcmp_size_63(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -48
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 44(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 40(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 36(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 32(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 28(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 24(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 20(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 16(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 12(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 8(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 4(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 24(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 28(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 24(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 28(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 47(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 51(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 55(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 59(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 31(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 35(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 39(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 43(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, t1, s2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a5, a5, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, a7, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 31(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 35(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 39(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 43(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a3, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, t0, s1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, a4, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 47(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 51(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 55(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 59(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, a6, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, a0, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, s6, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, s8, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, s4, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, s9, s2
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s5, s1
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, s7, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, s3, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, t3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, a6, t5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, s0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t0, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, a7, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t1, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a7, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t0, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a6, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v16
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 44(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 40(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 36(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 28(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 24(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 20(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 16(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 12(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 4(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 48
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_63:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 31(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 39(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 47(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 55(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 31(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 39(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 47(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 55(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, t3, t1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, t2, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, t4, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a2, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a5, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a3, a6
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a1, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 63
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v12, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v16, v8, v12
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v16
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
@@ -2627,270 +2719,24 @@ define i32 @bcmp_size_127(ptr %s1, ptr %s2) nounwind {
 ;
 ; CHECK-UNALIGNED-RV32-V-LABEL: bcmp_size_127:
 ; CHECK-UNALIGNED-RV32-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, -96
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw ra, 92(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s0, 88(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s1, 84(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s2, 80(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s3, 76(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s4, 72(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s5, 68(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s6, 64(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s7, 60(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s8, 56(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s9, 52(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s10, 48(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw s11, 44(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 32(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 36(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 40(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 44(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 0(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t1, 4(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 8(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 12(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 48(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t0, 52(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 56(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 60(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 16(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 20(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 24(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 28(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 12(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 60(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 16(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 20(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 24(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 28(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 32(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 36(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 40(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 44(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 40(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a3, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 36(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a4, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 32(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 56(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 48(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a6, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 28(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 4(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a7, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 24(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t0, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 20(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 0(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 8(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t1, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 16(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 107(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, t5, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    sw a2, 12(sp) # 4-byte Folded Spill
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 75(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t6, t6, s8
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 123(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s2, s2, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s0, s0, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t5, t4, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 83(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 87(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 91(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s1, s1, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 107(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t4, t3, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 91(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t3, t2, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 123(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t2, a5, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 75(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s5, s11, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s7, s8, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 87(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 83(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s3, s10, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 115(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s6, ra, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 115(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s4, s4, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 119(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a7, 119(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s10, s10, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 71(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 67(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 67(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a6, 71(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a4, 99(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a3, 99(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t1, a5, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 103(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 103(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor t0, a4, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a4, s9, s8
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a7, s11, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a6, ra, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a3, a5, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a5, 95(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 63(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 111(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a2, 79(a0)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 79(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 111(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a0, 63(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw a1, 95(a1)
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a2, a2, s11
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor s9, s9, ra
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a0, s8, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    xor a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t2, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, t3, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t4, s9
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, s1, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, t5, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a6, s0, a6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a7, s2, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, t6, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 12(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t2, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 16(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t1, t2, t1
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t2, 20(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t2, s10
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t3, t3, s4
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t4, 28(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t4, s6
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t5, 32(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t5, t5, s3
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw t6, 36(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, t6, s7
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 40(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    or s0, s0, s5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t6, s0, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t4, t5, t4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t2, t3, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or t0, t1, t0
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a4, a4, a7
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a6, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a2, a2, a5
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, t4, t6
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a5, t0, t2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a3, a3, a4
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a2
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a3
-; CHECK-UNALIGNED-RV32-V-NEXT:    or a0, a0, a1
+; CHECK-UNALIGNED-RV32-V-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV32-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV32-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV32-V-NEXT:    vcpop.m a0, v24
 ; CHECK-UNALIGNED-RV32-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw ra, 92(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s0, 88(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s1, 84(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s2, 80(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s3, 76(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s4, 72(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s5, 68(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s6, 64(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s7, 60(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s8, 56(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s9, 52(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s10, 48(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    lw s11, 44(sp) # 4-byte Folded Reload
-; CHECK-UNALIGNED-RV32-V-NEXT:    addi sp, sp, 96
 ; CHECK-UNALIGNED-RV32-V-NEXT:    ret
 ;
 ; CHECK-UNALIGNED-RV64-V-LABEL: bcmp_size_127:
 ; CHECK-UNALIGNED-RV64-V:       # %bb.0: # %entry
-; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, -96
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s0, 88(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s1, 80(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s2, 72(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s3, 64(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s4, 56(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s5, 48(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s6, 40(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s7, 32(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s8, 24(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s9, 16(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    sd s10, 8(sp) # 8-byte Folded Spill
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a2, 32(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a3, 40(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a4, 48(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a5, 56(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a6, 0(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a7, 8(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t0, 16(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t1, 24(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t2, 32(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 40(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 48(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 56(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t6, 0(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 8(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 16(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 24(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 95(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 103(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 111(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 119(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 63(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 71(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 79(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a0, 87(a0)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t1, t1, s2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a5, a5, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a7, a7, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t5, 63(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 71(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 79(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 87(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a3, a3, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t0, t0, s1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a4, a4, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t3, 95(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld t4, 103(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 111(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld a1, 119(a1)
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a6, a6, t6
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a2, a2, t2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a0, a0, s10
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor a1, s6, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t2, s8, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t4, s4, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t6, s9, s2
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor s0, s5, s1
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t5, s7, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    xor t3, s3, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, t3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a6, a6, t5
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a4, a4, s0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or t0, t0, t6
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, a3, t4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a7, a7, t2
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a5, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, t1, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a0, a1
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a1, a7, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a3, t0, a4
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a6, a2
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a1, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a2, a2, a3
-; CHECK-UNALIGNED-RV64-V-NEXT:    or a0, a2, a0
+; CHECK-UNALIGNED-RV64-V-NEXT:    li a2, 127
+; CHECK-UNALIGNED-RV64-V-NEXT:    vsetvli zero, a2, e8, m8, ta, ma
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v8, (a0)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vle8.v v16, (a1)
+; CHECK-UNALIGNED-RV64-V-NEXT:    vmsne.vv v24, v8, v16
+; CHECK-UNALIGNED-RV64-V-NEXT:    vcpop.m a0, v24
 ; CHECK-UNALIGNED-RV64-V-NEXT:    snez a0, a0
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s0, 88(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s1, 80(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s2, 72(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s3, 64(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s4, 56(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s5, 48(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s6, 40(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s7, 32(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s8, 24(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s9, 16(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    ld s10, 8(sp) # 8-byte Folded Reload
-; CHECK-UNALIGNED-RV64-V-NEXT:    addi sp, sp, 96
 ; CHECK-UNALIGNED-RV64-V-NEXT:    ret
 entry:
   %bcmp = call signext i32 @bcmp(ptr %s1, ptr %s2, iXLen 127)
-- 
cgit v1.2.3


From 59d6fbb8ffe03ceecfcc07ebe22e256c97ef70dd Mon Sep 17 00:00:00 2001
From: Kareem Ergawy <kareem.ergawy@amd.com>
Date: Wed, 18 Jun 2025 10:24:08 +0200
Subject: [flang][fir] Provide allocation block for `fir.local` when required
 (#144521)

Extends `fir::FirOpBuilder::getAllocaBlock()` to support `fir.local`.
This allows us to retrieve an allocation block when needed for
`fir.local`.
---
 flang/lib/Optimizer/Builder/FIRBuilder.cpp  |  3 +++
 flang/test/HLFIR/fir-local-alloca-block.fir | 34 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 flang/test/HLFIR/fir-local-alloca-block.fir

diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 584f3c8ee310..6ac87067f651 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -283,6 +283,9 @@ mlir::Block *fir::FirOpBuilder::getAllocaBlock() {
   if (auto doConcurentOp = getRegion().getParentOfType<fir::DoConcurrentOp>())
     return doConcurentOp.getBody();
 
+  if (auto firLocalOp = getRegion().getParentOfType<fir::LocalitySpecifierOp>())
+    return &getRegion().front();
+
   return getEntryBlock();
 }
 
diff --git a/flang/test/HLFIR/fir-local-alloca-block.fir b/flang/test/HLFIR/fir-local-alloca-block.fir
new file mode 100644
index 000000000000..9d76e86fec3d
--- /dev/null
+++ b/flang/test/HLFIR/fir-local-alloca-block.fir
@@ -0,0 +1,34 @@
+// Tests that `fir.local` ops are able to provide an alloca block when required.
+
+// RUN: fir-opt %s -convert-hlfir-to-fir | FileCheck %s
+
+fir.local {type = local_init} @localizer : !fir.box<!fir.array<1xi32>> copy {
+^bb0(%arg0: !fir.ref<!fir.box<!fir.array<1xi32>>>, %arg1: !fir.ref<!fir.box<!fir.array<1xi32>>>):
+  %0 = fir.load %arg0 : !fir.ref<!fir.box<!fir.array<1xi32>>>
+  hlfir.assign %0 to %arg1 : !fir.box<!fir.array<1xi32>>, !fir.ref<!fir.box<!fir.array<1xi32>>>
+  fir.yield(%arg1 : !fir.ref<!fir.box<!fir.array<1xi32>>>)
+}
+
+func.func @foo() {
+  %c1 = arith.constant 1 : index
+  %0 = fir.alloca !fir.box<!fir.array<1xi32>>
+  fir.do_concurrent {
+    fir.do_concurrent.loop (%arg0) = (%c1) to (%c1) step (%c1) local(@localizer %0 -> %arg1 : !fir.ref<!fir.box<!fir.array<1xi32>>>) {
+    }
+  }
+  return
+}
+
+// CHECK:  fir.local {type = local_init} @localizer : ![[TYPE:fir.box<!fir.array<1xi32>>]] copy {
+// CHECK:    ^bb0(%[[VAL_0:.*]]: !fir.ref<![[TYPE]]>, %[[VAL_1:.*]]: !fir.ref<![[TYPE]]>):
+// CHECK:      %[[VAL_2:.*]] = fir.alloca ![[TYPE]]
+// CHECK:      %[[VAL_3:.*]] = fir.load %[[VAL_0]] : !fir.ref<![[TYPE]]>
+// CHECK:      %[[VAL_4:.*]] = arith.constant 0 : index
+// CHECK:      %[[VAL_5:.*]]:3 = fir.box_dims %[[VAL_3]], %[[VAL_4]] : (![[TYPE]], index) -> (index, index, index)
+// CHECK:      %[[VAL_6:.*]] = fir.load %[[VAL_1]] : !fir.ref<![[TYPE]]>
+// CHECK:      fir.store %[[VAL_6]] to %[[VAL_2]] : !fir.ref<![[TYPE]]>
+// CHECK:      %[[VAL_10:.*]] = fir.convert %[[VAL_2]] : (!fir.ref<![[TYPE]]>) -> !fir.ref<!fir.box<none>>
+// CHECK:      %[[VAL_11:.*]] = fir.convert %[[VAL_3]] : (![[TYPE]]) -> !fir.box<none>
+// CHECK:      fir.call @_FortranAAssign(%[[VAL_10]], %[[VAL_11]], %{{.*}}, %{{.*}})
+// CHECK:      fir.yield(%[[VAL_1]] : !fir.ref<![[TYPE]]>)
+// CHECK:  }
-- 
cgit v1.2.3


From 255b55c602f73964262893859a543a115b278e21 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 18 Jun 2025 01:35:56 -0700
Subject: [GlobalOpt] Use cast instead of dyn_cast. NFC (#144634)

The dyn_cast was not checked for null, and the cast is guaranteed to
succeed by an earlier check.
---
 llvm/lib/Transforms/IPO/GlobalOpt.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Transforms/IPO/GlobalOpt.cpp b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
index 7db058638650..4a06e0fa619c 100644
--- a/llvm/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/llvm/lib/Transforms/IPO/GlobalOpt.cpp
@@ -2169,7 +2169,7 @@ static bool tryWidenGlobalArraysUsedByMemcpy(
 
     unsigned NumBytesToCopy = BytesToCopyOp->getZExtValue();
 
-    auto *Alloca = dyn_cast<AllocaInst>(CI->getArgOperand(0));
+    auto *Alloca = cast<AllocaInst>(CI->getArgOperand(0));
     uint64_t DZSize = Alloca->getAllocatedType()->getArrayNumElements();
     uint64_t SZSize = SourceDataArray->getType()->getNumElements();
     unsigned ElementByteWidth = SourceDataArray->getElementByteSize();
-- 
cgit v1.2.3


From 7ea7ccd24d603ceec6eb5194d98911e6ab7c0717 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 18 Jun 2025 10:50:17 +0200
Subject: [PowerPC][AIX] Specify pointer info and alignment for stack store
 (#144526)

When lowering call arguments to stack, specify a stack MPI, as well as
the stack alignment, instead of using the defaults (which would be an
unknown location with ABI alignment).

I believe the asm diffs are just changes in scheduling.
---
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp        |   4 +-
 llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll        | 366 +++++++++++----------
 llvm/test/CodeGen/PowerPC/aix-cc-abi.ll            | 176 +++++-----
 llvm/test/CodeGen/PowerPC/aix-cc-byval-mir.ll      |   4 +-
 .../test/CodeGen/PowerPC/aix-vec-arg-spills-mir.ll |  34 +-
 llvm/test/CodeGen/PowerPC/aix-vec-arg-spills.ll    |  38 +--
 .../CodeGen/PowerPC/aix-vector-vararg-caller.ll    |  26 +-
 .../PowerPC/aix-vector-vararg-fixed-caller.ll      |   8 +-
 8 files changed, 341 insertions(+), 315 deletions(-)

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 0f8e5e57c58b..f502d8570425 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -7767,7 +7767,9 @@ SDValue PPCTargetLowering::LowerCall_AIX(
           DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType());
       PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
       MemOpChains.push_back(
-          DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()));
+          DAG.getStore(Chain, dl, Arg, PtrOff,
+                       MachinePointerInfo::getStack(MF, VA.getLocMemOffset()),
+                       Subtarget.getFrameLowering()->getStackAlign()));
 
       continue;
     }
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll b/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
index aead5762d092..9ffb4fd5eae4 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-abi-mir.ll
@@ -474,14 +474,14 @@ define void @call_test_fpr_max() {
   ; 32BIT-NEXT:   renamable $r3 = LWZtoc @d1, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $r3 :: (dereferenceable load (s64) from @d1)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 128, 0, implicit-def dead $r1, implicit $r1
-  ; 32BIT-NEXT:   STFD renamable $f1, 120, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 112, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 104, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 96, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 88, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 80, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 72, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   STFD renamable $f1, 64, $r1 :: (store (s64))
+  ; 32BIT-NEXT:   STFD renamable $f1, 120, $r1 :: (store (s64) into stack + 120, basealign 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 112, $r1 :: (store (s64) into stack + 112, align 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 104, $r1 :: (store (s64) into stack + 104, basealign 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 96, $r1 :: (store (s64) into stack + 96, align 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 88, $r1 :: (store (s64) into stack + 88, basealign 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 80, $r1 :: (store (s64) into stack + 80, align 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 72, $r1 :: (store (s64) into stack + 72, basealign 16)
+  ; 32BIT-NEXT:   STFD renamable $f1, 64, $r1 :: (store (s64) into stack + 64, align 16)
   ; 32BIT-NEXT:   $f2 = COPY renamable $f1
   ; 32BIT-NEXT:   $f3 = COPY renamable $f1
   ; 32BIT-NEXT:   $f4 = COPY renamable $f1
@@ -494,7 +494,7 @@ define void @call_test_fpr_max() {
   ; 32BIT-NEXT:   $f11 = COPY renamable $f1
   ; 32BIT-NEXT:   $f12 = COPY renamable $f1
   ; 32BIT-NEXT:   $f13 = COPY renamable $f1
-  ; 32BIT-NEXT:   STFD renamable $f1, 56, $r1 :: (store (s64))
+  ; 32BIT-NEXT:   STFD renamable $f1, 56, $r1 :: (store (s64) into stack + 56, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_fpr_max>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit killed $f4, implicit killed $f5, implicit killed $f6, implicit killed $f7, implicit killed $f8, implicit killed $f9, implicit killed $f10, implicit killed $f11, implicit killed $f12, implicit killed $f13, implicit $r2, implicit-def $r1, implicit-def dead $f1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 128, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -504,10 +504,10 @@ define void @call_test_fpr_max() {
   ; 64BIT-NEXT:   renamable $x3 = LDtoc @d1, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $x3 :: (dereferenceable load (s64) from @d1)
   ; 64BIT-NEXT:   ADJCALLSTACKDOWN 152, 0, implicit-def dead $r1, implicit $r1
-  ; 64BIT-NEXT:   STFD renamable $f1, 144, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STFD renamable $f1, 136, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STFD renamable $f1, 128, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STFD renamable $f1, 120, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STFD renamable $f1, 144, $x1 :: (store (s64) into stack + 144, align 16)
+  ; 64BIT-NEXT:   STFD renamable $f1, 136, $x1 :: (store (s64) into stack + 136, basealign 16)
+  ; 64BIT-NEXT:   STFD renamable $f1, 128, $x1 :: (store (s64) into stack + 128, align 16)
+  ; 64BIT-NEXT:   STFD renamable $f1, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
   ; 64BIT-NEXT:   $f2 = COPY renamable $f1
   ; 64BIT-NEXT:   $f3 = COPY renamable $f1
   ; 64BIT-NEXT:   $f4 = COPY renamable $f1
@@ -520,7 +520,7 @@ define void @call_test_fpr_max() {
   ; 64BIT-NEXT:   $f11 = COPY renamable $f1
   ; 64BIT-NEXT:   $f12 = COPY renamable $f1
   ; 64BIT-NEXT:   $f13 = COPY renamable $f1
-  ; 64BIT-NEXT:   STFD renamable $f1, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STFD renamable $f1, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_fpr_max>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit killed $f2, implicit killed $f3, implicit killed $f4, implicit killed $f5, implicit killed $f6, implicit killed $f7, implicit killed $f8, implicit killed $f9, implicit killed $f10, implicit killed $f11, implicit killed $f12, implicit killed $f13, implicit $x2, implicit-def $r1, implicit-def dead $f1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 152, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -889,11 +889,11 @@ define void @call_test_stackarg_int() {
   ; 32BIT-NEXT:   renamable $r6 = LWZ 0, renamable $r3 :: (dereferenceable load (s32) from @lli, align 8)
   ; 32BIT-NEXT:   renamable $r3 = LWZ 4, killed renamable $r3 :: (dereferenceable load (s32) from @lli + 4, basealign 8)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 80, 0, implicit-def dead $r1, implicit $r1
-  ; 32BIT-NEXT:   STW renamable $r5, 76, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r3, 72, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r6, 68, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r5, 64, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r4, 60, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW renamable $r5, 76, $r1 :: (store (s32) into stack + 76, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 72, $r1 :: (store (s32) into stack + 72, align 8, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 68, $r1 :: (store (s32) into stack + 68, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r5, 64, $r1 :: (store (s32) into stack + 64, align 16)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
   ; 32BIT-NEXT:   $r3 = LI 1
   ; 32BIT-NEXT:   $r4 = LI 2
   ; 32BIT-NEXT:   $r5 = LI 3
@@ -902,7 +902,7 @@ define void @call_test_stackarg_int() {
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
   ; 32BIT-NEXT:   $r10 = LI 8
-  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_stackarg_int[PR]>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $r10, implicit $r2, implicit-def $r1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 80, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -926,11 +926,11 @@ define void @call_test_stackarg_int() {
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
   ; 64BIT-NEXT:   $x10 = LI8 8
-  ; 64BIT-NEXT:   STD killed renamable $x31, 136, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD renamable $x0, 144, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x0, 128, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x12, 120, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x11, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x31, 136, $x1 :: (store (s64) into stack + 136, basealign 16)
+  ; 64BIT-NEXT:   STD renamable $x0, 144, $x1 :: (store (s64) into stack + 144, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x0, 128, $x1 :: (store (s64) into stack + 128, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x12, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x11, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_stackarg_int[PR]>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x2, implicit-def $r1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 152, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -956,7 +956,11 @@ define void @call_test_stackarg_float() {
   ; 32BIT-NEXT:   renamable $f1 = LFS 0, killed renamable $r3 :: (dereferenceable load (s32) from @f)
   ; 32BIT-NEXT:   renamable $f2 = LFD 0, killed renamable $r4 :: (dereferenceable load (s64) from @d)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 68, 0, implicit-def dead $r1, implicit $r1
-  ; 32BIT-NEXT:   STFD renamable $f2, 60, $r1 :: (store (s64))
+  ; 32BIT-NEXT:   STFD renamable $f2, 0, %stack.0 :: (store (s64) into %stack.0)
+  ; 32BIT-NEXT:   STFS renamable $f1, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LWZ 4, %stack.0 :: (load (s32) from %stack.0 + 4)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 64, $r1 :: (store (s32) into stack + 64, align 16)
+  ; 32BIT-NEXT:   renamable $r11 = LWZ 0, %stack.0 :: (load (s32) from %stack.0, align 8)
   ; 32BIT-NEXT:   $r3 = LI 1
   ; 32BIT-NEXT:   $r4 = LI 2
   ; 32BIT-NEXT:   $r5 = LI 3
@@ -965,8 +969,8 @@ define void @call_test_stackarg_float() {
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
   ; 32BIT-NEXT:   $r10 = LI 8
-  ; 32BIT-NEXT:   STFS renamable $f1, 56, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_stackarg_float[PR]>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit killed $r5, implicit killed $r6, implicit killed $r7, implicit killed $r8, implicit killed $r9, implicit killed $r10, implicit $f1, implicit $f2, implicit $r2, implicit-def $r1
+  ; 32BIT-NEXT:   STW killed renamable $r11, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
+  ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_stackarg_float[PR]>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $r10, implicit $f1, implicit $f2, implicit $r2, implicit-def $r1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 68, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
   ;
@@ -977,7 +981,7 @@ define void @call_test_stackarg_float() {
   ; 64BIT-NEXT:   renamable $f1 = LFS 0, killed renamable $x3 :: (dereferenceable load (s32) from @f)
   ; 64BIT-NEXT:   renamable $f2 = LFD 0, killed renamable $x4 :: (dereferenceable load (s64) from @d)
   ; 64BIT-NEXT:   ADJCALLSTACKDOWN 128, 0, implicit-def dead $r1, implicit $r1
-  ; 64BIT-NEXT:   STFD renamable $f2, 120, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STFD renamable $f2, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
   ; 64BIT-NEXT:   $x3 = LI8 1
   ; 64BIT-NEXT:   $x4 = LI8 2
   ; 64BIT-NEXT:   $x5 = LI8 3
@@ -986,7 +990,7 @@ define void @call_test_stackarg_float() {
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
   ; 64BIT-NEXT:   $x10 = LI8 8
-  ; 64BIT-NEXT:   STFS renamable $f1, 112, $x1 :: (store (s32))
+  ; 64BIT-NEXT:   STFS renamable $f1, 112, $x1 :: (store (s32) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_stackarg_float[PR]>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit killed $x5, implicit killed $x6, implicit killed $x7, implicit killed $x8, implicit killed $x9, implicit killed $x10, implicit $f1, implicit $f2, implicit $x2, implicit-def $r1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 128, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1053,7 +1057,11 @@ define void @call_test_stackarg_float3() {
   ; 32BIT-NEXT:   renamable $r10 = LWZ 0, %stack.0 :: (load (s32) from %stack.0, align 8)
   ; 32BIT-NEXT:   renamable $f2 = LFS 0, killed renamable $r3 :: (dereferenceable load (s32) from @f)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 64, 0, implicit-def dead $r1, implicit $r1
-  ; 32BIT-NEXT:   STFS renamable $f2, 60, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STFD renamable $f1, 0, %stack.1 :: (store (s64) into %stack.1)
+  ; 32BIT-NEXT:   STFS renamable $f2, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LWZ 4, %stack.1 :: (load (s32) from %stack.1 + 4)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
+  ; 32BIT-NEXT:   renamable $r11 = LWZ 0, %stack.1 :: (load (s32) from %stack.1, align 8)
   ; 32BIT-NEXT:   $r3 = LI 1
   ; 32BIT-NEXT:   $r4 = LI 2
   ; 32BIT-NEXT:   $r5 = LI 3
@@ -1061,8 +1069,8 @@ define void @call_test_stackarg_float3() {
   ; 32BIT-NEXT:   $r7 = LI 5
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
-  ; 32BIT-NEXT:   STFD renamable $f1, 52, $r1 :: (store (s64))
-  ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_stackarg_float3[PR]>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit killed $r4, implicit killed $r5, implicit killed $r6, implicit killed $r7, implicit killed $r8, implicit killed $r9, implicit $f1, implicit $r10, implicit $f2, implicit $r2, implicit-def $r1
+  ; 32BIT-NEXT:   STW killed renamable $r11, 52, $r1 :: (store (s32) into stack + 52, basealign 16)
+  ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_stackarg_float3[PR]>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $f1, implicit $r10, implicit $f2, implicit $r2, implicit-def $r1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 64, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
   ;
@@ -1082,7 +1090,7 @@ define void @call_test_stackarg_float3() {
   ; 64BIT-NEXT:   $x7 = LI8 5
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
-  ; 64BIT-NEXT:   STFS renamable $f2, 112, $x1 :: (store (s32))
+  ; 64BIT-NEXT:   STFS renamable $f2, 112, $x1 :: (store (s32) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_stackarg_float3[PR]>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit killed $x4, implicit killed $x5, implicit killed $x6, implicit killed $x7, implicit killed $x8, implicit killed $x9, implicit $f1, implicit $x10, implicit $f2, implicit $x2, implicit-def $r1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 120, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1225,15 +1233,15 @@ define void @caller_ints_stack() {
   ; 32BIT-NEXT:   renamable $r9 = LBZ 0, killed renamable $r9 :: (dereferenceable load (s8) from @uc1)
   ; 32BIT-NEXT:   renamable $r12 = LWZ 0, killed renamable $r12 :: (dereferenceable load (s32) from @i1)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 96, 0, implicit-def dead $r1, implicit $r1
-  ; 32BIT-NEXT:   STW killed renamable $r12, 92, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r9, 88, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r8, 84, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r10, 80, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r7, 76, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r6, 72, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r5, 68, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r4, 64, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STW killed renamable $r3, 60, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r12, 92, $r1 :: (store (s32) into stack + 92, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r9, 88, $r1 :: (store (s32) into stack + 88, align 8, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r8, 84, $r1 :: (store (s32) into stack + 84, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r10, 80, $r1 :: (store (s32) into stack + 80, align 16)
+  ; 32BIT-NEXT:   STW killed renamable $r7, 76, $r1 :: (store (s32) into stack + 76, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 72, $r1 :: (store (s32) into stack + 72, align 8, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r5, 68, $r1 :: (store (s32) into stack + 68, basealign 16)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 64, $r1 :: (store (s32) into stack + 64, align 16)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
   ; 32BIT-NEXT:   $r3 = LI 1
   ; 32BIT-NEXT:   $r4 = LI 2
   ; 32BIT-NEXT:   $r5 = LI 3
@@ -1242,7 +1250,7 @@ define void @caller_ints_stack() {
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
   ; 32BIT-NEXT:   $r10 = LI 8
-  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_ints_stack>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $r10, implicit $r2, implicit-def $r1, implicit-def dead $r3, implicit-def dead $r4
   ; 32BIT-NEXT:   ADJCALLSTACKUP 96, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -1274,14 +1282,14 @@ define void @caller_ints_stack() {
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
   ; 64BIT-NEXT:   $x10 = LI8 8
-  ; 64BIT-NEXT:   STD killed renamable $x27, 168, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x30, 160, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x28, 152, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x31, 144, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x0, 136, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x12, 128, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x11, 120, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x29, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x27, 168, $x1 :: (store (s64) into stack + 168, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x30, 160, $x1 :: (store (s64) into stack + 160, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x28, 152, $x1 :: (store (s64) into stack + 152, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x31, 144, $x1 :: (store (s64) into stack + 144, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x0, 136, $x1 :: (store (s64) into stack + 136, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x12, 128, $x1 :: (store (s64) into stack + 128, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x11, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x29, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_ints_stack>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x2, implicit-def $r1, implicit-def dead $x3
   ; 64BIT-NEXT:   ADJCALLSTACKUP 176, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1333,7 +1341,7 @@ define void @call_test_i1_stack() {
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
   ; 32BIT-NEXT:   $r10 = LI 8
-  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r11, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_i1_stack>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $r10, implicit $r2, implicit-def $r1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 60, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -1350,7 +1358,7 @@ define void @call_test_i1_stack() {
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
   ; 64BIT-NEXT:   $x10 = LI8 8
-  ; 64BIT-NEXT:   STD killed renamable $x11, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x11, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_i1_stack>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $x2, implicit-def $r1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 120, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1441,88 +1449,92 @@ define void @caller_fpr_stack() {
   ; 32BIT-NEXT:   renamable $r3 = LWZtoc @d15, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $r4 = LWZtoc @f14, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $f0 = LFD 0, killed renamable $r3 :: (dereferenceable load (s64) from @d15)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc @f16, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $r3 = LWZ 0, killed renamable $r4 :: (dereferenceable load (s32) from @f14)
-  ; 32BIT-NEXT:   renamable $r4 = LWZ 0, killed renamable $r5 :: (dereferenceable load (s32) from @f16)
+  ; 32BIT-NEXT:   renamable $r3 = LWZtoc @f16, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $r4 = LWZ 0, killed renamable $r4 :: (dereferenceable load (s32) from @f14)
+  ; 32BIT-NEXT:   renamable $r3 = LWZ 0, killed renamable $r3 :: (dereferenceable load (s32) from @f16)
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 144, 0, implicit-def dead $r1, implicit $r1
+  ; 32BIT-NEXT:   STFD killed renamable $f0, 0, %stack.0 :: (store (s64) into %stack.0)
   ; 32BIT-NEXT:   renamable $r5 = LI 0
   ; 32BIT-NEXT:   renamable $r6 = LIS 16352
-  ; 32BIT-NEXT:   STW killed renamable $r5, 60, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
-  ; 32BIT-NEXT:   renamable $r5 = LIS 13107
-  ; 32BIT-NEXT:   STW killed renamable $r6, 56, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LIS 16355
-  ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 13107
-  ; 32BIT-NEXT:   STW killed renamable $r5, 68, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 140, $r1 :: (store (s32) into stack + 140, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LIS 13107
+  ; 32BIT-NEXT:   STW killed renamable $r4, 128, $r1 :: (store (s32) into stack + 128, align 16)
+  ; 32BIT-NEXT:   renamable $r4 = LIS 16355
+  ; 32BIT-NEXT:   STW killed renamable $r5, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
   ; 32BIT-NEXT:   renamable $r5 = LIS 26214
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 13107
-  ; 32BIT-NEXT:   STW killed renamable $r6, 64, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = LIS 16358
+  ; 32BIT-NEXT:   renamable $r3 = ORI killed renamable $r3, 13107
+  ; 32BIT-NEXT:   STW killed renamable $r3, 68, $r1 :: (store (s32) into stack + 68, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LIS 39321
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 13107
+  ; 32BIT-NEXT:   STW killed renamable $r4, 64, $r1 :: (store (s32) into stack + 64, align 16)
+  ; 32BIT-NEXT:   renamable $r4 = LIS 16361
   ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 26214
-  ; 32BIT-NEXT:   STW killed renamable $r5, 76, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
-  ; 32BIT-NEXT:   renamable $r5 = LIS 39321
+  ; 32BIT-NEXT:   STW killed renamable $r5, 76, $r1 :: (store (s32) into stack + 76, basealign 16)
+  ; 32BIT-NEXT:   renamable $r5 = LIS 52428
   ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 26214
-  ; 32BIT-NEXT:   STW killed renamable $r6, 72, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LIS 16361
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r6, 80, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LIS 52428
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 52429
-  ; 32BIT-NEXT:   STW killed renamable $r6, 92, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 72, $r1 :: (store (s32) into stack + 72, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = LIS 16364
-  ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 39322
-  ; 32BIT-NEXT:   STW renamable $r5, 84, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 52428
-  ; 32BIT-NEXT:   STW killed renamable $r6, 88, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LIS 16313
-  ; 32BIT-NEXT:   STW killed renamable $r5, 100, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 39321
+  ; 32BIT-NEXT:   STW killed renamable $r4, 80, $r1 :: (store (s32) into stack + 80, align 16)
+  ; 32BIT-NEXT:   renamable $r4 = LIS 16313
+  ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 52429
+  ; 32BIT-NEXT:   STW killed renamable $r5, 92, $r1 :: (store (s32) into stack + 92, basealign 16)
   ; 32BIT-NEXT:   renamable $r5 = LIS 49807
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r6, 96, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   renamable $r3 = ORI killed renamable $r3, 39322
+  ; 32BIT-NEXT:   STW renamable $r3, 84, $r1 :: (store (s32) into stack + 84, basealign 16)
+  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 52428
+  ; 32BIT-NEXT:   STW killed renamable $r6, 88, $r1 :: (store (s32) into stack + 88, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = LIS 16316
+  ; 32BIT-NEXT:   STW killed renamable $r3, 100, $r1 :: (store (s32) into stack + 100, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LIS 60293
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 39321
+  ; 32BIT-NEXT:   STW killed renamable $r4, 96, $r1 :: (store (s32) into stack + 96, align 16)
+  ; 32BIT-NEXT:   renamable $r4 = LIS 16318
   ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 23593
-  ; 32BIT-NEXT:   STW killed renamable $r5, 108, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
-  ; 32BIT-NEXT:   renamable $r5 = LIS 60293
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 10485
-  ; 32BIT-NEXT:   STW killed renamable $r6, 104, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LIS 16318
-  ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 7864
-  ; 32BIT-NEXT:   STW killed renamable $r5, 116, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r5, 108, $r1 :: (store (s32) into stack + 108, basealign 16)
   ; 32BIT-NEXT:   renamable $r5 = LIS 2621
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 47185
-  ; 32BIT-NEXT:   STW killed renamable $r6, 112, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 10485
+  ; 32BIT-NEXT:   STW killed renamable $r6, 104, $r1 :: (store (s32) into stack + 104, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = LIS 16320
-  ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r5, 28836
-  ; 32BIT-NEXT:   STW killed renamable $r5, 124, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.0, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r6, 41943
-  ; 32BIT-NEXT:   STW killed renamable $r6, 120, $r1 :: (store (s32), align 8)
-  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.1, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f2 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.2, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f3 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.3, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f4 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.4, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f6 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r3 = ORI killed renamable $r3, 7864
+  ; 32BIT-NEXT:   STW killed renamable $r3, 116, $r1 :: (store (s32) into stack + 116, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.0, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 47185
+  ; 32BIT-NEXT:   STW killed renamable $r4, 112, $r1 :: (store (s32) into stack + 112, align 16)
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r5, 28836
+  ; 32BIT-NEXT:   STW killed renamable $r4, 124, $r1 :: (store (s32) into stack + 124, basealign 16)
+  ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r6, 41943
+  ; 32BIT-NEXT:   STW killed renamable $r4, 120, $r1 :: (store (s32) into stack + 120, align 8, basealign 16)
+  ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.1, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $r5 = LWZ 4, %stack.0 :: (load (s32) from %stack.0 + 4)
+  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.2, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f2 = LFD 0, killed renamable $r3 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.3, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f3 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.4, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f4 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.5, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f7 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.6, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f6 = LFD 0, killed renamable $r3 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.6, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f7 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.7, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $f8 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.7, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f9 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.8, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.9, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f11 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
-  ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.10, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f12 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.8, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f9 = LFD 0, killed renamable $r3 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.9, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.10, $r2 :: (load (s32) from got)
+  ; 32BIT-NEXT:   renamable $f11 = LFD 0, killed renamable $r6 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $r6 = LWZtoc %const.11, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   renamable $f13 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $f12 = LFD 0, killed renamable $r3 :: (load (s64) from constant-pool)
+  ; 32BIT-NEXT:   renamable $f13 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $f5 = LFS 0, killed renamable $r6 :: (load (s32) from constant-pool)
-  ; 32BIT-NEXT:   STW killed renamable $r4, 140, $r1 :: (store (s32))
-  ; 32BIT-NEXT:   STFD killed renamable $f0, 132, $r1 :: (store (s64))
+  ; 32BIT-NEXT:   STW killed renamable $r5, 136, $r1 :: (store (s32) into stack + 136, align 8, basealign 16)
+  ; 32BIT-NEXT:   renamable $r3 = LWZ 0, %stack.0 :: (load (s32) from %stack.0, align 8)
   ; 32BIT-NEXT:   $f10 = COPY renamable $f1
-  ; 32BIT-NEXT:   STW killed renamable $r3, 128, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 132, $r1 :: (store (s32) into stack + 132, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .test_fpr_stack>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit $f5, implicit $f6, implicit $f7, implicit $f8, implicit $f9, implicit killed $f10, implicit $f11, implicit $f12, implicit $f13, implicit $r2, implicit-def $r1, implicit-def dead $f1
   ; 32BIT-NEXT:   ADJCALLSTACKUP 144, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -1537,7 +1549,7 @@ define void @caller_fpr_stack() {
   ; 64BIT-NEXT:   renamable $r5 = LWZ 0, killed renamable $x5 :: (dereferenceable load (s32) from @f16)
   ; 64BIT-NEXT:   ADJCALLSTACKDOWN 176, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   renamable $x6 = LDtocCPT %const.0, $x2 :: (load (s64) from got)
-  ; 64BIT-NEXT:   STW killed renamable $r5, 168, $x1 :: (store (s32))
+  ; 64BIT-NEXT:   STW killed renamable $r5, 168, $x1 :: (store (s32) into stack + 168, align 8, basealign 16)
   ; 64BIT-NEXT:   renamable $x5 = LDtocCPT %const.1, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   renamable $x7 = LDtocCPT %const.2, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   renamable $f2 = LFD 0, killed renamable $x6 :: (load (s64) from constant-pool)
@@ -1549,7 +1561,7 @@ define void @caller_fpr_stack() {
   ; 64BIT-NEXT:   renamable $f6 = LFD 0, killed renamable $x6 :: (load (s64) from constant-pool)
   ; 64BIT-NEXT:   renamable $x6 = LDtocCPT %const.6, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   renamable $f7 = LFD 0, killed renamable $x5 :: (load (s64) from constant-pool)
-  ; 64BIT-NEXT:   STD killed renamable $x4, 160, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x4, 160, $x1 :: (store (s64) into stack + 160, align 16)
   ; 64BIT-NEXT:   renamable $x4 = LDtocCPT %const.7, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   renamable $f8 = LFD 0, killed renamable $x7 :: (load (s64) from constant-pool)
   ; 64BIT-NEXT:   renamable $x5 = LIS8 16320
@@ -1588,12 +1600,12 @@ define void @caller_fpr_stack() {
   ; 64BIT-NEXT:   renamable $x8 = ORIS8 killed renamable $x8, 52428
   ; 64BIT-NEXT:   renamable $x8 = ORI8 killed renamable $x8, 52429
   ; 64BIT-NEXT:   $f10 = COPY renamable $f1
-  ; 64BIT-NEXT:   STW killed renamable $r3, 152, $x1 :: (store (s32))
-  ; 64BIT-NEXT:   STD killed renamable $x5, 144, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x6, 136, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x4, 128, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x7, 120, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x8, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STW killed renamable $r3, 152, $x1 :: (store (s32) into stack + 152, align 8, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x5, 144, $x1 :: (store (s64) into stack + 144, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x6, 136, $x1 :: (store (s64) into stack + 136, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x4, 128, $x1 :: (store (s64) into stack + 128, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x7, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x8, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .test_fpr_stack>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit $f5, implicit $f6, implicit $f7, implicit $f8, implicit $f9, implicit killed $f10, implicit $f11, implicit $f12, implicit $f13, implicit $x2, implicit-def $r1, implicit-def dead $f1
   ; 64BIT-NEXT:   ADJCALLSTACKUP 176, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1688,16 +1700,16 @@ define void @caller_mix() {
   ; 32BIT: bb.0.entry:
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 84, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   renamable $r3 = LI 60
-  ; 32BIT-NEXT:   STW killed renamable $r3, 80, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 80, $r1 :: (store (s32) into stack + 80, align 16)
   ; 32BIT-NEXT:   renamable $r3 = LI 50
-  ; 32BIT-NEXT:   STW killed renamable $r3, 76, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 76, $r1 :: (store (s32) into stack + 76, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LI 40
-  ; 32BIT-NEXT:   STW killed renamable $r3, 72, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 72, $r1 :: (store (s32) into stack + 72, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LI 0
   ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.0, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   STW killed renamable $r3, 64, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 64, $r1 :: (store (s32) into stack + 64, align 16)
   ; 32BIT-NEXT:   renamable $r3 = LI 2
-  ; 32BIT-NEXT:   STW killed renamable $r3, 60, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.1, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.2, $r2 :: (load (s32) from got)
@@ -1706,10 +1718,10 @@ define void @caller_mix() {
   ; 32BIT-NEXT:   renamable $f3 = LFD 0, killed renamable $r4 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $f4 = LFD 0, killed renamable $r3 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $r3 = LI 1
-  ; 32BIT-NEXT:   STW killed renamable $r3, 56, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LIS 457
   ; 32BIT-NEXT:   renamable $r3 = ORI killed renamable $r3, 50048
-  ; 32BIT-NEXT:   STW killed renamable $r3, 68, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 68, $r1 :: (store (s32) into stack + 68, basealign 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .mix_callee>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit $r2, implicit-def $r1, implicit-def dead $r3
   ; 32BIT-NEXT:   ADJCALLSTACKUP 84, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -1732,8 +1744,8 @@ define void @caller_mix() {
   ; 64BIT-NEXT:   $x7 = LI8 1
   ; 64BIT-NEXT:   $x8 = LI8 2
   ; 64BIT-NEXT:   $x10 = LI8 40
-  ; 64BIT-NEXT:   STD killed renamable $x4, 120, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x5, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x4, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x5, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .mix_callee>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit killed $x7, implicit killed $x8, implicit $x9, implicit killed $x10, implicit $x2, implicit-def $r1, implicit-def dead $x3
   ; 64BIT-NEXT:   ADJCALLSTACKUP 128, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
@@ -1855,60 +1867,60 @@ define void @caller_mix() {
   ; 32BIT-NEXT:   renamable $r8 = LIS 16329
   ; 32BIT-NEXT:   renamable $r9 = LIS 13107
   ; 32BIT-NEXT:   renamable $r10 = LIS 16339
-  ; 32BIT-NEXT:   STW renamable $r3, 92, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r3, 92, $r1 :: (store (s32) into stack + 92, basealign 16)
   ; 32BIT-NEXT:   renamable $r11 = LIS 16345
-  ; 32BIT-NEXT:   STW killed renamable $r4, 88, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 88, $r1 :: (store (s32) into stack + 88, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = LIS 16355
-  ; 32BIT-NEXT:   STW killed renamable $r3, 132, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 132, $r1 :: (store (s32) into stack + 132, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LIS 26214
-  ; 32BIT-NEXT:   STW killed renamable $r5, 128, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r5, 128, $r1 :: (store (s32) into stack + 128, align 16)
   ; 32BIT-NEXT:   renamable $r5 = ORI killed renamable $r6, 39322
-  ; 32BIT-NEXT:   STW renamable $r5, 60, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r5, 60, $r1 :: (store (s32) into stack + 60, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r7, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r6, 56, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = LIS 16358
-  ; 32BIT-NEXT:   STW renamable $r5, 68, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r5, 68, $r1 :: (store (s32) into stack + 68, basealign 16)
   ; 32BIT-NEXT:   renamable $r7 = ORI killed renamable $r8, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r7, 64, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r7, 64, $r1 :: (store (s32) into stack + 64, align 16)
   ; 32BIT-NEXT:   renamable $r7 = ORI killed renamable $r9, 13107
-  ; 32BIT-NEXT:   STW renamable $r7, 76, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r7, 76, $r1 :: (store (s32) into stack + 76, basealign 16)
   ; 32BIT-NEXT:   renamable $r8 = ORI killed renamable $r10, 13107
-  ; 32BIT-NEXT:   STW killed renamable $r8, 72, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r8, 72, $r1 :: (store (s32) into stack + 72, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r8 = LIS 16361
-  ; 32BIT-NEXT:   STW renamable $r5, 84, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r5, 84, $r1 :: (store (s32) into stack + 84, basealign 16)
   ; 32BIT-NEXT:   renamable $r9 = ORI killed renamable $r11, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r9, 80, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r9, 80, $r1 :: (store (s32) into stack + 80, align 16)
   ; 32BIT-NEXT:   renamable $r9 = LIS 52428
-  ; 32BIT-NEXT:   STW renamable $r7, 100, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r7, 100, $r1 :: (store (s32) into stack + 100, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 13107
-  ; 32BIT-NEXT:   STW killed renamable $r4, 96, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 96, $r1 :: (store (s32) into stack + 96, align 16)
   ; 32BIT-NEXT:   renamable $r3 = ORI killed renamable $r3, 26214
-  ; 32BIT-NEXT:   STW renamable $r3, 108, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r3, 108, $r1 :: (store (s32) into stack + 108, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r6, 26214
-  ; 32BIT-NEXT:   STW killed renamable $r4, 104, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 104, $r1 :: (store (s32) into stack + 104, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = LIS 16364
-  ; 32BIT-NEXT:   STW renamable $r5, 116, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r5, 116, $r1 :: (store (s32) into stack + 116, basealign 16)
   ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r8, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r6, 112, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 112, $r1 :: (store (s32) into stack + 112, align 16)
   ; 32BIT-NEXT:   renamable $r6 = ORI killed renamable $r9, 52429
-  ; 32BIT-NEXT:   STW renamable $r6, 124, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW renamable $r6, 124, $r1 :: (store (s32) into stack + 124, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 52428
-  ; 32BIT-NEXT:   STW killed renamable $r4, 120, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 120, $r1 :: (store (s32) into stack + 120, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = LIS 16369
-  ; 32BIT-NEXT:   STW killed renamable $r5, 140, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r5, 140, $r1 :: (store (s32) into stack + 140, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 39321
-  ; 32BIT-NEXT:   STW killed renamable $r4, 136, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 136, $r1 :: (store (s32) into stack + 136, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = LIS 16371
-  ; 32BIT-NEXT:   STW killed renamable $r7, 148, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r7, 148, $r1 :: (store (s32) into stack + 148, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 13107
-  ; 32BIT-NEXT:   STW killed renamable $r4, 144, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 144, $r1 :: (store (s32) into stack + 144, align 16)
   ; 32BIT-NEXT:   renamable $r4 = LIS 16372
   ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.0, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   STW killed renamable $r6, 156, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r6, 156, $r1 :: (store (s32) into stack + 156, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = ORI killed renamable $r4, 52428
-  ; 32BIT-NEXT:   STW killed renamable $r4, 152, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r4, 152, $r1 :: (store (s32) into stack + 152, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r4 = LWZtoc %const.1, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   STW killed renamable $r3, 164, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed renamable $r3, 164, $r1 :: (store (s32) into stack + 164, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LWZtoc %const.2, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   renamable $f1 = LFD 0, killed renamable $r5 :: (load (s64) from constant-pool)
   ; 32BIT-NEXT:   renamable $r5 = LWZtoc %const.3, $r2 :: (load (s32) from got)
@@ -1943,7 +1955,7 @@ define void @caller_mix() {
   ; 32BIT-NEXT:   $r8 = LI 6
   ; 32BIT-NEXT:   $r9 = LI 7
   ; 32BIT-NEXT:   $r10 = LI 8
-  ; 32BIT-NEXT:   STW killed renamable $r11, 160, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed renamable $r11, 160, $r1 :: (store (s32) into stack + 160, align 16)
   ; 32BIT-NEXT:   BL_NOP <mcsymbol .mix_floats>, csr_aix32, implicit-def dead $lr, implicit $rm, implicit $r3, implicit $r4, implicit $r5, implicit $r6, implicit $r7, implicit $r8, implicit $r9, implicit $r10, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit $f5, implicit $f6, implicit $f7, implicit $f8, implicit $f9, implicit $f10, implicit $f11, implicit $f12, implicit $f13, implicit $r2, implicit-def $r1, implicit-def dead $r3
   ; 32BIT-NEXT:   ADJCALLSTACKUP 168, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   BLR implicit $lr, implicit $rm
@@ -2044,20 +2056,20 @@ define void @caller_mix() {
   ; 64BIT-NEXT:   $x8 = LI8 6
   ; 64BIT-NEXT:   $x9 = LI8 7
   ; 64BIT-NEXT:   $x10 = LI8 8
-  ; 64BIT-NEXT:   STD killed renamable $x29, 184, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x28, 144, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x11, 216, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x12, 200, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x0, 160, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x31, 152, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x30, 128, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x27, 208, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x25, 192, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x24, 176, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x23, 168, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x22, 136, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x21, 120, $x1 :: (store (s64))
-  ; 64BIT-NEXT:   STD killed renamable $x20, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x29, 184, $x1 :: (store (s64) into stack + 184, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x28, 144, $x1 :: (store (s64) into stack + 144, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x11, 216, $x1 :: (store (s64) into stack + 216, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x12, 200, $x1 :: (store (s64) into stack + 200, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x0, 160, $x1 :: (store (s64) into stack + 160, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x31, 152, $x1 :: (store (s64) into stack + 152, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x30, 128, $x1 :: (store (s64) into stack + 128, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x27, 208, $x1 :: (store (s64) into stack + 208, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x25, 192, $x1 :: (store (s64) into stack + 192, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x24, 176, $x1 :: (store (s64) into stack + 176, align 16)
+  ; 64BIT-NEXT:   STD killed renamable $x23, 168, $x1 :: (store (s64) into stack + 168, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x22, 136, $x1 :: (store (s64) into stack + 136, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x21, 120, $x1 :: (store (s64) into stack + 120, basealign 16)
+  ; 64BIT-NEXT:   STD killed renamable $x20, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   BL8_NOP <mcsymbol .mix_floats>, csr_ppc64, implicit-def dead $lr8, implicit $rm, implicit $x3, implicit $x4, implicit $x5, implicit $x6, implicit $x7, implicit $x8, implicit $x9, implicit $x10, implicit $f1, implicit $f2, implicit $f3, implicit $f4, implicit $f5, implicit $f6, implicit $f7, implicit $f8, implicit $f9, implicit $f10, implicit $f11, implicit $f12, implicit $f13, implicit $x2, implicit-def $r1, implicit-def dead $x3
   ; 64BIT-NEXT:   ADJCALLSTACKUP 224, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   BLR8 implicit $lr8, implicit $rm
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
index 8f33f5ef863e..03770d22d9f4 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-abi.ll
@@ -1012,18 +1012,22 @@ define void @call_test_stackarg_float() {
 ; ASM32PWR4-NEXT:    lwz 3, L..C8(2) # @f
 ; ASM32PWR4-NEXT:    stw 0, 88(1)
 ; ASM32PWR4-NEXT:    li 4, 2
-; ASM32PWR4-NEXT:    li 5, 3
 ; ASM32PWR4-NEXT:    li 6, 4
 ; ASM32PWR4-NEXT:    li 7, 5
+; ASM32PWR4-NEXT:    li 8, 6
 ; ASM32PWR4-NEXT:    lfs 1, 0(3)
 ; ASM32PWR4-NEXT:    lwz 3, L..C9(2) # @d
-; ASM32PWR4-NEXT:    li 8, 6
 ; ASM32PWR4-NEXT:    li 9, 7
+; ASM32PWR4-NEXT:    li 10, 8
 ; ASM32PWR4-NEXT:    lfd 2, 0(3)
 ; ASM32PWR4-NEXT:    li 3, 1
-; ASM32PWR4-NEXT:    li 10, 8
-; ASM32PWR4-NEXT:    stfd 2, 60(1)
+; ASM32PWR4-NEXT:    stfd 2, 72(1)
+; ASM32PWR4-NEXT:    lwz 5, 76(1)
+; ASM32PWR4-NEXT:    lwz 11, 72(1)
+; ASM32PWR4-NEXT:    stw 5, 64(1)
+; ASM32PWR4-NEXT:    li 5, 3
 ; ASM32PWR4-NEXT:    stfs 1, 56(1)
+; ASM32PWR4-NEXT:    stw 11, 60(1)
 ; ASM32PWR4-NEXT:    bl .test_stackarg_float[PR]
 ; ASM32PWR4-NEXT:    nop
 ; ASM32PWR4-NEXT:    addi 1, 1, 80
@@ -1126,20 +1130,24 @@ define void @call_test_stackarg_float3() {
 ; ASM32PWR4-NEXT:    stwu 1, -80(1)
 ; ASM32PWR4-NEXT:    lwz 3, L..C9(2) # @d
 ; ASM32PWR4-NEXT:    stw 0, 88(1)
-; ASM32PWR4-NEXT:    li 4, 2
 ; ASM32PWR4-NEXT:    li 5, 3
 ; ASM32PWR4-NEXT:    li 6, 4
 ; ASM32PWR4-NEXT:    li 7, 5
+; ASM32PWR4-NEXT:    li 8, 6
 ; ASM32PWR4-NEXT:    lfd 1, 0(3)
 ; ASM32PWR4-NEXT:    lwz 3, L..C8(2) # @f
-; ASM32PWR4-NEXT:    li 8, 6
 ; ASM32PWR4-NEXT:    li 9, 7
 ; ASM32PWR4-NEXT:    stfd 1, 72(1)
-; ASM32PWR4-NEXT:    lwz 10, 72(1)
 ; ASM32PWR4-NEXT:    lfs 2, 0(3)
 ; ASM32PWR4-NEXT:    li 3, 1
+; ASM32PWR4-NEXT:    stfd 1, 64(1)
+; ASM32PWR4-NEXT:    lwz 4, 68(1)
+; ASM32PWR4-NEXT:    lwz 10, 72(1)
+; ASM32PWR4-NEXT:    lwz 11, 64(1)
+; ASM32PWR4-NEXT:    stw 4, 56(1)
+; ASM32PWR4-NEXT:    li 4, 2
 ; ASM32PWR4-NEXT:    stfs 2, 60(1)
-; ASM32PWR4-NEXT:    stfd 1, 52(1)
+; ASM32PWR4-NEXT:    stw 11, 52(1)
 ; ASM32PWR4-NEXT:    bl .test_stackarg_float3[PR]
 ; ASM32PWR4-NEXT:    nop
 ; ASM32PWR4-NEXT:    addi 1, 1, 80
@@ -1562,95 +1570,99 @@ define void @caller_fpr_stack() {
 ; ASM32PWR4-LABEL: caller_fpr_stack:
 ; ASM32PWR4:       # %bb.0: # %entry
 ; ASM32PWR4-NEXT:    mflr 0
-; ASM32PWR4-NEXT:    stwu 1, -144(1)
+; ASM32PWR4-NEXT:    stwu 1, -160(1)
 ; ASM32PWR4-NEXT:    lwz 3, L..C19(2) # @d15
-; ASM32PWR4-NEXT:    lwz 4, L..C20(2) # @f14
-; ASM32PWR4-NEXT:    lwz 5, L..C21(2) # @f16
-; ASM32PWR4-NEXT:    stw 0, 152(1)
-; ASM32PWR4-NEXT:    lis 6, 16361
-; ASM32PWR4-NEXT:    ori 6, 6, 39321
+; ASM32PWR4-NEXT:    stw 0, 168(1)
+; ASM32PWR4-NEXT:    lwz 5, L..C20(2) # %const.1
+; ASM32PWR4-NEXT:    lwz 4, L..C21(2) # @f14
 ; ASM32PWR4-NEXT:    lfd 0, 0(3)
-; ASM32PWR4-NEXT:    lwz 3, 0(4)
-; ASM32PWR4-NEXT:    lwz 4, 0(5)
-; ASM32PWR4-NEXT:    li 5, 0
-; ASM32PWR4-NEXT:    stw 5, 60(1)
-; ASM32PWR4-NEXT:    lis 5, 16352
-; ASM32PWR4-NEXT:    stw 5, 56(1)
-; ASM32PWR4-NEXT:    lis 5, 13107
-; ASM32PWR4-NEXT:    ori 5, 5, 13107
-; ASM32PWR4-NEXT:    stw 5, 68(1)
-; ASM32PWR4-NEXT:    lis 5, 16355
-; ASM32PWR4-NEXT:    ori 5, 5, 13107
-; ASM32PWR4-NEXT:    stw 5, 64(1)
-; ASM32PWR4-NEXT:    lis 5, 26214
-; ASM32PWR4-NEXT:    ori 5, 5, 26214
-; ASM32PWR4-NEXT:    stw 5, 76(1)
-; ASM32PWR4-NEXT:    lis 5, 16358
-; ASM32PWR4-NEXT:    ori 5, 5, 26214
-; ASM32PWR4-NEXT:    stw 5, 72(1)
-; ASM32PWR4-NEXT:    lis 5, -26215
-; ASM32PWR4-NEXT:    ori 5, 5, 39322
-; ASM32PWR4-NEXT:    stw 5, 84(1)
-; ASM32PWR4-NEXT:    stw 5, 100(1)
-; ASM32PWR4-NEXT:    lis 5, 16313
-; ASM32PWR4-NEXT:    ori 5, 5, 39321
-; ASM32PWR4-NEXT:    stw 5, 96(1)
-; ASM32PWR4-NEXT:    lis 5, -15729
-; ASM32PWR4-NEXT:    ori 5, 5, 23593
-; ASM32PWR4-NEXT:    stw 5, 108(1)
-; ASM32PWR4-NEXT:    lis 5, 16316
-; ASM32PWR4-NEXT:    ori 5, 5, 10485
-; ASM32PWR4-NEXT:    stw 5, 104(1)
-; ASM32PWR4-NEXT:    lis 5, -5243
-; ASM32PWR4-NEXT:    ori 5, 5, 7864
-; ASM32PWR4-NEXT:    stw 5, 116(1)
-; ASM32PWR4-NEXT:    lis 5, 16318
-; ASM32PWR4-NEXT:    ori 5, 5, 47185
-; ASM32PWR4-NEXT:    stw 6, 80(1)
-; ASM32PWR4-NEXT:    lis 6, -13108
-; ASM32PWR4-NEXT:    ori 6, 6, 52429
-; ASM32PWR4-NEXT:    stw 5, 112(1)
-; ASM32PWR4-NEXT:    lis 5, 2621
-; ASM32PWR4-NEXT:    ori 5, 5, 28836
-; ASM32PWR4-NEXT:    stw 6, 92(1)
-; ASM32PWR4-NEXT:    lis 6, 16364
-; ASM32PWR4-NEXT:    ori 6, 6, 52428
-; ASM32PWR4-NEXT:    stw 5, 124(1)
-; ASM32PWR4-NEXT:    lis 5, 16320
-; ASM32PWR4-NEXT:    ori 5, 5, 41943
-; ASM32PWR4-NEXT:    stw 6, 88(1)
-; ASM32PWR4-NEXT:    lwz 6, L..C22(2) # %const.0
-; ASM32PWR4-NEXT:    stw 5, 120(1)
-; ASM32PWR4-NEXT:    lwz 5, L..C23(2) # %const.1
-; ASM32PWR4-NEXT:    lfd 2, 0(6)
-; ASM32PWR4-NEXT:    lwz 6, L..C24(2) # %const.2
+; ASM32PWR4-NEXT:    lwz 3, L..C22(2) # @f16
+; ASM32PWR4-NEXT:    lwz 3, 0(3)
+; ASM32PWR4-NEXT:    stw 3, 140(1)
+; ASM32PWR4-NEXT:    li 3, 0
+; ASM32PWR4-NEXT:    stw 3, 60(1)
+; ASM32PWR4-NEXT:    lis 3, 16352
+; ASM32PWR4-NEXT:    stw 3, 56(1)
+; ASM32PWR4-NEXT:    lis 3, 13107
+; ASM32PWR4-NEXT:    ori 3, 3, 13107
+; ASM32PWR4-NEXT:    stw 3, 68(1)
+; ASM32PWR4-NEXT:    lis 3, 16355
+; ASM32PWR4-NEXT:    ori 3, 3, 13107
+; ASM32PWR4-NEXT:    stw 3, 64(1)
+; ASM32PWR4-NEXT:    lis 3, 26214
+; ASM32PWR4-NEXT:    ori 3, 3, 26214
+; ASM32PWR4-NEXT:    stw 3, 76(1)
+; ASM32PWR4-NEXT:    lis 3, 16358
+; ASM32PWR4-NEXT:    ori 3, 3, 26214
+; ASM32PWR4-NEXT:    stw 3, 72(1)
+; ASM32PWR4-NEXT:    lis 3, -26215
+; ASM32PWR4-NEXT:    ori 3, 3, 39322
+; ASM32PWR4-NEXT:    stw 3, 84(1)
+; ASM32PWR4-NEXT:    stw 3, 100(1)
+; ASM32PWR4-NEXT:    lis 3, 16313
+; ASM32PWR4-NEXT:    ori 3, 3, 39321
+; ASM32PWR4-NEXT:    stw 3, 96(1)
+; ASM32PWR4-NEXT:    lis 3, -15729
+; ASM32PWR4-NEXT:    ori 3, 3, 23593
+; ASM32PWR4-NEXT:    stw 3, 108(1)
+; ASM32PWR4-NEXT:    lis 3, 16316
+; ASM32PWR4-NEXT:    ori 3, 3, 10485
+; ASM32PWR4-NEXT:    stw 3, 104(1)
+; ASM32PWR4-NEXT:    lis 3, -5243
+; ASM32PWR4-NEXT:    ori 3, 3, 7864
+; ASM32PWR4-NEXT:    stw 3, 116(1)
+; ASM32PWR4-NEXT:    lis 3, 16318
+; ASM32PWR4-NEXT:    ori 3, 3, 47185
+; ASM32PWR4-NEXT:    stw 3, 112(1)
+; ASM32PWR4-NEXT:    lis 3, 2621
+; ASM32PWR4-NEXT:    ori 3, 3, 28836
+; ASM32PWR4-NEXT:    stw 3, 124(1)
+; ASM32PWR4-NEXT:    lis 3, 16320
+; ASM32PWR4-NEXT:    ori 3, 3, 41943
+; ASM32PWR4-NEXT:    stw 3, 120(1)
+; ASM32PWR4-NEXT:    lwz 3, L..C23(2) # %const.0
+; ASM32PWR4-NEXT:    lfd 2, 0(3)
+; ASM32PWR4-NEXT:    lwz 3, L..C24(2) # %const.2
 ; ASM32PWR4-NEXT:    lfd 3, 0(5)
 ; ASM32PWR4-NEXT:    lwz 5, L..C25(2) # %const.3
-; ASM32PWR4-NEXT:    lfd 4, 0(6)
-; ASM32PWR4-NEXT:    lwz 6, L..C26(2) # %const.4
+; ASM32PWR4-NEXT:    lfd 4, 0(3)
+; ASM32PWR4-NEXT:    lwz 3, L..C26(2) # %const.4
 ; ASM32PWR4-NEXT:    lfd 6, 0(5)
 ; ASM32PWR4-NEXT:    lwz 5, L..C27(2) # %const.5
-; ASM32PWR4-NEXT:    lfd 7, 0(6)
-; ASM32PWR4-NEXT:    lwz 6, L..C28(2) # %const.6
+; ASM32PWR4-NEXT:    lwz 4, 0(4)
+; ASM32PWR4-NEXT:    lfd 7, 0(3)
+; ASM32PWR4-NEXT:    lwz 3, L..C28(2) # %const.6
 ; ASM32PWR4-NEXT:    lfd 8, 0(5)
 ; ASM32PWR4-NEXT:    lwz 5, L..C29(2) # %const.7
-; ASM32PWR4-NEXT:    lfd 9, 0(6)
-; ASM32PWR4-NEXT:    lwz 6, L..C30(2) # %const.8
+; ASM32PWR4-NEXT:    stw 4, 128(1)
+; ASM32PWR4-NEXT:    lis 4, 16361
+; ASM32PWR4-NEXT:    ori 4, 4, 39321
+; ASM32PWR4-NEXT:    lfd 9, 0(3)
+; ASM32PWR4-NEXT:    lwz 3, L..C30(2) # %const.8
 ; ASM32PWR4-NEXT:    lfd 1, 0(5)
 ; ASM32PWR4-NEXT:    lwz 5, L..C31(2) # %const.9
-; ASM32PWR4-NEXT:    lfd 11, 0(6)
-; ASM32PWR4-NEXT:    lwz 6, L..C32(2) # %const.10
+; ASM32PWR4-NEXT:    stw 4, 80(1)
+; ASM32PWR4-NEXT:    lis 4, -13108
 ; ASM32PWR4-NEXT:    fmr 10, 1
+; ASM32PWR4-NEXT:    ori 4, 4, 52429
+; ASM32PWR4-NEXT:    lfd 11, 0(3)
+; ASM32PWR4-NEXT:    lwz 3, L..C32(2) # %const.10
 ; ASM32PWR4-NEXT:    lfd 12, 0(5)
 ; ASM32PWR4-NEXT:    lwz 5, L..C33(2) # %const.11
-; ASM32PWR4-NEXT:    lfd 13, 0(6)
+; ASM32PWR4-NEXT:    stw 4, 92(1)
+; ASM32PWR4-NEXT:    lis 4, 16364
+; ASM32PWR4-NEXT:    ori 4, 4, 52428
+; ASM32PWR4-NEXT:    stfd 0, 152(1)
+; ASM32PWR4-NEXT:    stw 4, 88(1)
+; ASM32PWR4-NEXT:    lwz 4, 156(1)
+; ASM32PWR4-NEXT:    lfd 13, 0(3)
 ; ASM32PWR4-NEXT:    lfs 5, 0(5)
-; ASM32PWR4-NEXT:    stfd 0, 132(1)
-; ASM32PWR4-NEXT:    stw 4, 140(1)
-; ASM32PWR4-NEXT:    stw 3, 128(1)
+; ASM32PWR4-NEXT:    lwz 3, 152(1)
+; ASM32PWR4-NEXT:    stw 4, 136(1)
+; ASM32PWR4-NEXT:    stw 3, 132(1)
 ; ASM32PWR4-NEXT:    bl .test_fpr_stack
 ; ASM32PWR4-NEXT:    nop
-; ASM32PWR4-NEXT:    addi 1, 1, 144
+; ASM32PWR4-NEXT:    addi 1, 1, 160
 ; ASM32PWR4-NEXT:    lwz 0, 8(1)
 ; ASM32PWR4-NEXT:    mtlr 0
 ; ASM32PWR4-NEXT:    blr
@@ -1667,7 +1679,6 @@ define void @caller_fpr_stack() {
 ; ASM64PWR4-NEXT:    lis 7, 16313
 ; ASM64PWR4-NEXT:    lwz 3, 0(3)
 ; ASM64PWR4-NEXT:    ld 4, 0(4)
-; ASM64PWR4-NEXT:    lwz 5, 0(5)
 ; ASM64PWR4-NEXT:    stw 3, 152(1)
 ; ASM64PWR4-NEXT:    ld 3, L..C22(2) # %const.0
 ; ASM64PWR4-NEXT:    std 4, 160(1)
@@ -1686,6 +1697,7 @@ define void @caller_fpr_stack() {
 ; ASM64PWR4-NEXT:    ld 4, L..C29(2) # %const.7
 ; ASM64PWR4-NEXT:    lfd 9, 0(3)
 ; ASM64PWR4-NEXT:    ld 3, L..C30(2) # %const.8
+; ASM64PWR4-NEXT:    lwz 5, 0(5)
 ; ASM64PWR4-NEXT:    lfd 1, 0(4)
 ; ASM64PWR4-NEXT:    lis 4, 16320
 ; ASM64PWR4-NEXT:    ori 4, 4, 41943
diff --git a/llvm/test/CodeGen/PowerPC/aix-cc-byval-mir.ll b/llvm/test/CodeGen/PowerPC/aix-cc-byval-mir.ll
index 67800df6ed4b..95fed680e696 100644
--- a/llvm/test/CodeGen/PowerPC/aix-cc-byval-mir.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-cc-byval-mir.ll
@@ -134,7 +134,7 @@ define void @call_test_byval_3Byte() {
   ; 32BIT-NEXT:   ADJCALLSTACKDOWN 60, 0, implicit-def dead $r1, implicit $r1
   ; 32BIT-NEXT:   renamable $r3 = LI 42
   ; 32BIT-NEXT:   renamable $r4 = LWZtoc @gS3, $r2 :: (load (s32) from got)
-  ; 32BIT-NEXT:   STW killed renamable $r3, 56, $r1 :: (store (s32))
+  ; 32BIT-NEXT:   STW killed renamable $r3, 56, $r1 :: (store (s32) into stack + 56, align 8, basealign 16)
   ; 32BIT-NEXT:   renamable $r3 = LBZ 2, renamable $r4 :: (load (s8))
   ; 32BIT-NEXT:   renamable $r4 = LHZ 0, killed renamable $r4 :: (load (s16))
   ; 32BIT-NEXT:   renamable $r10 = RLWINM killed renamable $r3, 8, 16, 23
@@ -155,7 +155,7 @@ define void @call_test_byval_3Byte() {
   ; 64BIT-NEXT:   ADJCALLSTACKDOWN 120, 0, implicit-def dead $r1, implicit $r1
   ; 64BIT-NEXT:   renamable $x3 = LI8 42
   ; 64BIT-NEXT:   renamable $x4 = LDtoc @gS3, $x2 :: (load (s64) from got)
-  ; 64BIT-NEXT:   STD killed renamable $x3, 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed renamable $x3, 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   renamable $x3 = LBZ8 2, renamable $x4 :: (load (s8))
   ; 64BIT-NEXT:   renamable $x4 = LHZ8 0, killed renamable $x4 :: (load (s16))
   ; 64BIT-NEXT:   renamable $x10 = RLDIC killed renamable $x3, 40, 16
diff --git a/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills-mir.ll b/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills-mir.ll
index 7c45958a1c2f..7ee854c2ae22 100644
--- a/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills-mir.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills-mir.ll
@@ -34,17 +34,17 @@ define double @caller() {
   ; MIR32-NEXT:   STXVW4X renamable $vsl0, $r1, killed renamable $r3 :: (store (s128), align 8)
   ; MIR32-NEXT:   renamable $r3 = LI 104
   ; MIR32-NEXT:   STXVW4X renamable $vsl0, $r1, killed renamable $r4 :: (store (s128), align 8)
-  ; MIR32-NEXT:   renamable $r4 = LI 88
   ; MIR32-NEXT:   STXVW4X renamable $vsl0, $r1, killed renamable $r3 :: (store (s128), align 8)
-  ; MIR32-NEXT:   STXVW4X renamable $vsl0, $r1, killed renamable $r4 :: (store (s128), align 8)
-  ; MIR32-NEXT:   renamable $r3 = LI 72
-  ; MIR32-NEXT:   renamable $r4 = LWZtoc %const.0, $r2 :: (load (s32) from got)
-  ; MIR32-NEXT:   STXVW4X killed renamable $vsl0, $r1, killed renamable $r3 :: (store (s128), align 8)
+  ; MIR32-NEXT:   renamable $r3 = LI 88
+  ; MIR32-NEXT:   renamable $r4 = LI 72
+  ; MIR32-NEXT:   STXVW4X renamable $vsl0, $r1, killed renamable $r3 :: (store (s128), align 8)
   ; MIR32-NEXT:   renamable $r3 = LI 48
-  ; MIR32-NEXT:   renamable $vsl0 = LXVD2X $zero, killed renamable $r4 :: (load (s128) from constant-pool)
+  ; MIR32-NEXT:   STXVW4X killed renamable $vsl0, $r1, killed renamable $r4 :: (store (s128), align 8)
   ; MIR32-NEXT:   renamable $r4 = LI 512
-  ; MIR32-NEXT:   STXVD2X killed renamable $vsl0, $r1, killed renamable $r3 :: (store (s128))
-  ; MIR32-NEXT:   STW killed renamable $r4, 152, $r1 :: (store (s32))
+  ; MIR32-NEXT:   STW killed renamable $r4, 152, $r1 :: (store (s32) into stack + 152, align 8, basealign 16)
+  ; MIR32-NEXT:   renamable $r4 = LWZtoc %const.0, $r2 :: (load (s32) from got)
+  ; MIR32-NEXT:   renamable $vsl0 = LXVD2X $zero, killed renamable $r4 :: (load (s128) from constant-pool)
+  ; MIR32-NEXT:   STXVD2X killed renamable $vsl0, $r1, killed renamable $r3 :: (store (s128) into stack + 48)
   ; MIR32-NEXT:   $f1 = XXLXORdpz
   ; MIR32-NEXT:   $f2 = XXLXORdpz
   ; MIR32-NEXT:   $v2 = XXLXORz
@@ -92,18 +92,18 @@ define double @caller() {
   ; MIR64-NEXT:   ADJCALLSTACKDOWN 224, 0, implicit-def dead $r1, implicit $r1
   ; MIR64-NEXT:   renamable $vsl0 = XXLXORz
   ; MIR64-NEXT:   renamable $x3 = LI8 160
-  ; MIR64-NEXT:   STXVW4X renamable $vsl0, $x1, killed renamable $x3 :: (store (s128), align 8)
+  ; MIR64-NEXT:   STXVW4X renamable $vsl0, $x1, killed renamable $x3 :: (store (s128))
   ; MIR64-NEXT:   renamable $x3 = LI8 144
-  ; MIR64-NEXT:   STXVW4X renamable $vsl0, $x1, killed renamable $x3 :: (store (s128), align 8)
+  ; MIR64-NEXT:   STXVW4X renamable $vsl0, $x1, killed renamable $x3 :: (store (s128))
   ; MIR64-NEXT:   renamable $x3 = LI8 128
-  ; MIR64-NEXT:   STXVW4X killed renamable $vsl0, $x1, killed renamable $x3 :: (store (s128), align 8)
-  ; MIR64-NEXT:   renamable $x3 = LDtocCPT %const.0, $x2 :: (load (s64) from got)
-  ; MIR64-NEXT:   renamable $vsl0 = LXVD2X $zero8, killed renamable $x3 :: (load (s128) from constant-pool)
-  ; MIR64-NEXT:   renamable $x3 = LI8 80
-  ; MIR64-NEXT:   STXVD2X killed renamable $vsl0, $x1, killed renamable $x3 :: (store (s128))
+  ; MIR64-NEXT:   STXVW4X killed renamable $vsl0, $x1, killed renamable $x3 :: (store (s128))
   ; MIR64-NEXT:   renamable $x3 = LI8 512
-  ; MIR64-NEXT:   STD killed renamable $x3, 184, $x1 :: (store (s64))
-  ; MIR64-NEXT:   STD killed renamable $x4, 176, $x1 :: (store (s64))
+  ; MIR64-NEXT:   STD killed renamable $x3, 184, $x1 :: (store (s64) into stack + 184, basealign 16)
+  ; MIR64-NEXT:   renamable $x3 = LI8 80
+  ; MIR64-NEXT:   STD killed renamable $x4, 176, $x1 :: (store (s64) into stack + 176, align 16)
+  ; MIR64-NEXT:   renamable $x4 = LDtocCPT %const.0, $x2 :: (load (s64) from got)
+  ; MIR64-NEXT:   renamable $vsl0 = LXVD2X $zero8, killed renamable $x4 :: (load (s128) from constant-pool)
+  ; MIR64-NEXT:   STXVD2X killed renamable $vsl0, $x1, killed renamable $x3 :: (store (s128) into stack + 80)
   ; MIR64-NEXT:   $f1 = XXLXORdpz
   ; MIR64-NEXT:   $f2 = XXLXORdpz
   ; MIR64-NEXT:   $v2 = XXLXORz
diff --git a/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills.ll b/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills.ll
index 66f88b4e3d5a..294f074807f1 100644
--- a/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-vec-arg-spills.ll
@@ -25,42 +25,42 @@ define double @caller() {
 ; 32BIT-NEXT:    stw 3, 184(1)
 ; 32BIT-NEXT:    stw 3, 176(1)
 ; 32BIT-NEXT:    stw 4, 172(1)
-; 32BIT-NEXT:    lis 4, 16368
 ; 32BIT-NEXT:    stw 3, 168(1)
+; 32BIT-NEXT:    lis 4, 16368
 ; 32BIT-NEXT:    stw 3, 160(1)
-; 32BIT-NEXT:    stw 4, 164(1)
 ; 32BIT-NEXT:    stw 3, 156(1)
 ; 32BIT-NEXT:    li 3, 136
+; 32BIT-NEXT:    stw 4, 164(1)
 ; 32BIT-NEXT:    li 4, 120
-; 32BIT-NEXT:    xxlxor 2, 2, 2
 ; 32BIT-NEXT:    stxvw4x 0, 1, 3
 ; 32BIT-NEXT:    li 3, 104
 ; 32BIT-NEXT:    stxvw4x 0, 1, 4
-; 32BIT-NEXT:    li 4, 88
+; 32BIT-NEXT:    stxvw4x 0, 1, 3
+; 32BIT-NEXT:    li 3, 88
+; 32BIT-NEXT:    li 4, 72
+; 32BIT-NEXT:    xxlxor 2, 2, 2
 ; 32BIT-NEXT:    stxvw4x 0, 1, 3
 ; 32BIT-NEXT:    stxvw4x 0, 1, 4
+; 32BIT-NEXT:    li 4, 512
+; 32BIT-NEXT:    stw 4, 152(1)
 ; 32BIT-NEXT:    lwz 4, L..C0(2) # %const.0
-; 32BIT-NEXT:    li 3, 72
-; 32BIT-NEXT:    stxvw4x 0, 1, 3
 ; 32BIT-NEXT:    li 3, 48
 ; 32BIT-NEXT:    xxlxor 34, 34, 34
 ; 32BIT-NEXT:    xxlxor 35, 35, 35
-; 32BIT-NEXT:    lxvd2x 0, 0, 4
-; 32BIT-NEXT:    li 4, 512
 ; 32BIT-NEXT:    xxlxor 36, 36, 36
 ; 32BIT-NEXT:    xxlxor 37, 37, 37
 ; 32BIT-NEXT:    xxlxor 38, 38, 38
+; 32BIT-NEXT:    lxvd2x 0, 0, 4
 ; 32BIT-NEXT:    xxlxor 39, 39, 39
 ; 32BIT-NEXT:    xxlxor 40, 40, 40
+; 32BIT-NEXT:    li 4, 256
 ; 32BIT-NEXT:    xxlxor 41, 41, 41
 ; 32BIT-NEXT:    xxlxor 42, 42, 42
-; 32BIT-NEXT:    stxvd2x 0, 1, 3
-; 32BIT-NEXT:    stw 4, 152(1)
-; 32BIT-NEXT:    li 3, 128
-; 32BIT-NEXT:    li 4, 256
 ; 32BIT-NEXT:    xxlxor 43, 43, 43
 ; 32BIT-NEXT:    xxlxor 44, 44, 44
 ; 32BIT-NEXT:    xxlxor 45, 45, 45
+; 32BIT-NEXT:    stxvd2x 0, 1, 3
+; 32BIT-NEXT:    li 3, 128
 ; 32BIT-NEXT:    xxlxor 3, 3, 3
 ; 32BIT-NEXT:    xxlxor 4, 4, 4
 ; 32BIT-NEXT:    xxlxor 5, 5, 5
@@ -114,23 +114,23 @@ define double @caller() {
 ; 64BIT-NEXT:    li 3, 128
 ; 64BIT-NEXT:    xxlxor 43, 43, 43
 ; 64BIT-NEXT:    stxvw4x 0, 1, 3
-; 64BIT-NEXT:    ld 3, L..C0(2) # %const.0
+; 64BIT-NEXT:    std 4, 176(1)
+; 64BIT-NEXT:    ld 4, L..C0(2) # %const.0
+; 64BIT-NEXT:    li 3, 512
 ; 64BIT-NEXT:    xxlxor 44, 44, 44
 ; 64BIT-NEXT:    xxlxor 45, 45, 45
-; 64BIT-NEXT:    lxvd2x 0, 0, 3
+; 64BIT-NEXT:    lxvd2x 0, 0, 4
+; 64BIT-NEXT:    std 3, 184(1)
 ; 64BIT-NEXT:    li 3, 80
+; 64BIT-NEXT:    li 4, 256
 ; 64BIT-NEXT:    xxlxor 3, 3, 3
 ; 64BIT-NEXT:    xxlxor 4, 4, 4
 ; 64BIT-NEXT:    xxlxor 5, 5, 5
 ; 64BIT-NEXT:    stxvd2x 0, 1, 3
-; 64BIT-NEXT:    li 3, 512
-; 64BIT-NEXT:    std 4, 176(1)
-; 64BIT-NEXT:    li 4, 256
+; 64BIT-NEXT:    li 3, 128
 ; 64BIT-NEXT:    xxlxor 6, 6, 6
 ; 64BIT-NEXT:    xxlxor 7, 7, 7
 ; 64BIT-NEXT:    xxlxor 8, 8, 8
-; 64BIT-NEXT:    std 3, 184(1)
-; 64BIT-NEXT:    li 3, 128
 ; 64BIT-NEXT:    xxlxor 9, 9, 9
 ; 64BIT-NEXT:    xxlxor 10, 10, 10
 ; 64BIT-NEXT:    xxlxor 11, 11, 11
diff --git a/llvm/test/CodeGen/PowerPC/aix-vector-vararg-caller.ll b/llvm/test/CodeGen/PowerPC/aix-vector-vararg-caller.ll
index 4697a093e5d6..0ba345fb5275 100644
--- a/llvm/test/CodeGen/PowerPC/aix-vector-vararg-caller.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-vector-vararg-caller.ll
@@ -23,31 +23,31 @@ define <4 x i32> @caller() {
   ; 32BIT-NEXT:   [[LWZtoc2:%[0-9]+]]:gprc = LWZtoc %const.2, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X2:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc2]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI2:%[0-9]+]]:gprc = LI 160
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X2]], $r1, killed [[LI2]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X2]], $r1, killed [[LI2]] :: (store (s128) into stack + 160)
   ; 32BIT-NEXT:   [[LWZtoc3:%[0-9]+]]:gprc = LWZtoc %const.3, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X3:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc3]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI3:%[0-9]+]]:gprc = LI 144
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X3]], $r1, killed [[LI3]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X3]], $r1, killed [[LI3]] :: (store (s128) into stack + 144)
   ; 32BIT-NEXT:   [[LWZtoc4:%[0-9]+]]:gprc = LWZtoc %const.4, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X4:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc4]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI4:%[0-9]+]]:gprc = LI 128
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X4]], $r1, killed [[LI4]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X4]], $r1, killed [[LI4]] :: (store (s128) into stack + 128)
   ; 32BIT-NEXT:   [[LWZtoc5:%[0-9]+]]:gprc = LWZtoc %const.5, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X5:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc5]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI5:%[0-9]+]]:gprc = LI 112
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X5]], $r1, killed [[LI5]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X5]], $r1, killed [[LI5]] :: (store (s128) into stack + 112)
   ; 32BIT-NEXT:   [[LWZtoc6:%[0-9]+]]:gprc = LWZtoc %const.6, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X6:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc6]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI6:%[0-9]+]]:gprc = LI 96
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X6]], $r1, killed [[LI6]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X6]], $r1, killed [[LI6]] :: (store (s128) into stack + 96)
   ; 32BIT-NEXT:   [[LWZtoc7:%[0-9]+]]:gprc = LWZtoc %const.7, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X7:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc7]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI7:%[0-9]+]]:gprc = LI 80
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X7]], $r1, killed [[LI7]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X7]], $r1, killed [[LI7]] :: (store (s128) into stack + 80)
   ; 32BIT-NEXT:   [[LWZtoc8:%[0-9]+]]:gprc = LWZtoc %const.8, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X8:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc8]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI8:%[0-9]+]]:gprc = LI 64
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X8]], $r1, killed [[LI8]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X8]], $r1, killed [[LI8]] :: (store (s128) into stack + 64)
   ; 32BIT-NEXT:   [[LWZ:%[0-9]+]]:gprc = LWZ 52, $r1 :: (load (s32))
   ; 32BIT-NEXT:   [[LWZ1:%[0-9]+]]:gprc = LWZ 48, $r1 :: (load (s32))
   ; 32BIT-NEXT:   [[LWZ2:%[0-9]+]]:gprc = LWZ 44, $r1 :: (load (s32))
@@ -86,27 +86,27 @@ define <4 x i32> @caller() {
   ; 64BIT-NEXT:   [[LDtocCPT3:%[0-9]+]]:g8rc = LDtocCPT %const.3, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X3:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT3]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_3:%[0-9]+]]:g8rc = LI8 192
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X3]], $x1, killed [[LI8_3]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X3]], $x1, killed [[LI8_3]] :: (store (s128) into stack + 192)
   ; 64BIT-NEXT:   [[LDtocCPT4:%[0-9]+]]:g8rc = LDtocCPT %const.4, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X4:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT4]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_4:%[0-9]+]]:g8rc = LI8 176
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X4]], $x1, killed [[LI8_4]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X4]], $x1, killed [[LI8_4]] :: (store (s128) into stack + 176)
   ; 64BIT-NEXT:   [[LDtocCPT5:%[0-9]+]]:g8rc = LDtocCPT %const.5, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X5:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT5]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_5:%[0-9]+]]:g8rc = LI8 160
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X5]], $x1, killed [[LI8_5]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X5]], $x1, killed [[LI8_5]] :: (store (s128) into stack + 160)
   ; 64BIT-NEXT:   [[LDtocCPT6:%[0-9]+]]:g8rc = LDtocCPT %const.6, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X6:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT6]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_6:%[0-9]+]]:g8rc = LI8 144
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X6]], $x1, killed [[LI8_6]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X6]], $x1, killed [[LI8_6]] :: (store (s128) into stack + 144)
   ; 64BIT-NEXT:   [[LDtocCPT7:%[0-9]+]]:g8rc = LDtocCPT %const.7, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X7:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT7]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_7:%[0-9]+]]:g8rc = LI8 128
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X7]], $x1, killed [[LI8_7]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X7]], $x1, killed [[LI8_7]] :: (store (s128) into stack + 128)
   ; 64BIT-NEXT:   [[LDtocCPT8:%[0-9]+]]:g8rc = LDtocCPT %const.8, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X8:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT8]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LI8_8:%[0-9]+]]:g8rc = LI8 112
-  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X8]], $x1, killed [[LI8_8]] :: (store (s128))
+  ; 64BIT-NEXT:   STXVW4X killed [[LXVW4X8]], $x1, killed [[LI8_8]] :: (store (s128) into stack + 112)
   ; 64BIT-NEXT:   [[LD:%[0-9]+]]:g8rc = LD 104, $x1 :: (load (s64))
   ; 64BIT-NEXT:   [[LD1:%[0-9]+]]:g8rc = LD 96, $x1 :: (load (s64))
   ; 64BIT-NEXT:   [[LD2:%[0-9]+]]:g8rc = LD 88, $x1 :: (load (s64))
diff --git a/llvm/test/CodeGen/PowerPC/aix-vector-vararg-fixed-caller.ll b/llvm/test/CodeGen/PowerPC/aix-vector-vararg-fixed-caller.ll
index fad275f58cd0..b39a94e17563 100644
--- a/llvm/test/CodeGen/PowerPC/aix-vector-vararg-fixed-caller.ll
+++ b/llvm/test/CodeGen/PowerPC/aix-vector-vararg-fixed-caller.ll
@@ -16,13 +16,13 @@ define void @caller() {
   ; 32BIT-NEXT:   [[LWZtoc:%[0-9]+]]:gprc = LWZtoc %const.0, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LI:%[0-9]+]]:gprc = LI 64
-  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X]], $r1, killed [[LI]] :: (store (s128))
+  ; 32BIT-NEXT:   STXVW4X killed [[LXVW4X]], $r1, killed [[LI]] :: (store (s128) into stack + 64)
   ; 32BIT-NEXT:   [[LIS:%[0-9]+]]:gprc = LIS 38314
   ; 32BIT-NEXT:   [[ORI:%[0-9]+]]:gprc = ORI killed [[LIS]], 63376
-  ; 32BIT-NEXT:   STW killed [[ORI]], 84, $r1 :: (store (s32) into unknown-address + 4, basealign 8)
+  ; 32BIT-NEXT:   STW killed [[ORI]], 84, $r1 :: (store (s32) into stack + 84, basealign 16)
   ; 32BIT-NEXT:   [[LIS1:%[0-9]+]]:gprc = LIS 16389
   ; 32BIT-NEXT:   [[ORI1:%[0-9]+]]:gprc = ORI killed [[LIS1]], 48905
-  ; 32BIT-NEXT:   STW killed [[ORI1]], 80, $r1 :: (store (s32), align 8)
+  ; 32BIT-NEXT:   STW killed [[ORI1]], 80, $r1 :: (store (s32) into stack + 80, align 16)
   ; 32BIT-NEXT:   [[LWZtoc1:%[0-9]+]]:gprc = LWZtoc %const.1, $r2 :: (load (s32) from got)
   ; 32BIT-NEXT:   [[LXVW4X1:%[0-9]+]]:vsrc = LXVW4X $zero, killed [[LWZtoc1]] :: (load (s128) from constant-pool)
   ; 32BIT-NEXT:   [[LWZtoc2:%[0-9]+]]:gprc_and_gprc_nor0 = LWZtoc %const.2, $r2 :: (load (s32) from got)
@@ -57,7 +57,7 @@ define void @caller() {
   ; 64BIT-NEXT:   [[RLDIC:%[0-9]+]]:g8rc = RLDIC killed [[ORI8_]], 32, 1
   ; 64BIT-NEXT:   [[ORIS8_:%[0-9]+]]:g8rc = ORIS8 killed [[RLDIC]], 38314
   ; 64BIT-NEXT:   [[ORI8_1:%[0-9]+]]:g8rc = ORI8 killed [[ORIS8_]], 63376
-  ; 64BIT-NEXT:   STD killed [[ORI8_1]], 112, $x1 :: (store (s64))
+  ; 64BIT-NEXT:   STD killed [[ORI8_1]], 112, $x1 :: (store (s64) into stack + 112, align 16)
   ; 64BIT-NEXT:   [[LDtocCPT1:%[0-9]+]]:g8rc = LDtocCPT %const.1, $x2 :: (load (s64) from got)
   ; 64BIT-NEXT:   [[LXVW4X1:%[0-9]+]]:vsrc = LXVW4X $zero8, killed [[LDtocCPT1]] :: (load (s128) from constant-pool)
   ; 64BIT-NEXT:   [[LD:%[0-9]+]]:g8rc = LD 104, $x1 :: (load (s64))
-- 
cgit v1.2.3


From c16dc63b44ae039f2ac123a8ffbc90031767d00b Mon Sep 17 00:00:00 2001
From: Mikael Holmen <mikael.holmen@ericsson.com>
Date: Wed, 18 Jun 2025 09:23:25 +0200
Subject: [OMPIRBuilder] Fix gcc -Wparentheses warning [NFC]

Without this gcc warned like
 /repo/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp:7559:68: warning: suggest parentheses around '&&' within '||' [-Wparentheses]
  7559 |         NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2) &&
       |         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^~
  7560 |             "Wrong number of arguments for StaleCI when shareds are present");
       |             ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 7cbbbff511c8..ddc9c5392f92 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -7558,10 +7558,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitTargetTask(
     // StaleCI is exactly OffloadingArraysToPrivatize.size() + 2
     const unsigned int NumStaleCIArgs = StaleCI->arg_size();
     bool HasShareds = NumStaleCIArgs > OffloadingArraysToPrivatize.size() + 1;
-    assert(
-        !HasShareds ||
-        NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2) &&
-            "Wrong number of arguments for StaleCI when shareds are present");
+    assert((!HasShareds ||
+            NumStaleCIArgs == (OffloadingArraysToPrivatize.size() + 2)) &&
+           "Wrong number of arguments for StaleCI when shareds are present");
     int SharedArgOperandNo =
         HasShareds ? OffloadingArraysToPrivatize.size() + 1 : 0;
 
-- 
cgit v1.2.3


From 669627d0c77ed8408358bc8c5973255fe28a36ea Mon Sep 17 00:00:00 2001
From: Philipp Jung <philippjung2010@live.de>
Date: Wed, 18 Jun 2025 11:02:53 +0200
Subject: Add check 'cppcoreguidelines-use-enum-class' (#138282)

Warn on non-class enum definitions as suggested by the Core Guidelines:
https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-class
---
 .../clang-tidy/cppcoreguidelines/CMakeLists.txt    |  1 +
 .../CppCoreGuidelinesTidyModule.cpp                |  3 ++
 .../cppcoreguidelines/UseEnumClassCheck.cpp        | 42 +++++++++++++++
 .../cppcoreguidelines/UseEnumClassCheck.h          | 40 ++++++++++++++
 clang-tools-extra/docs/ReleaseNotes.rst            |  6 +++
 .../checks/cppcoreguidelines/use-enum-class.rst    | 35 ++++++++++++
 clang-tools-extra/docs/clang-tidy/checks/list.rst  |  1 +
 .../checkers/cppcoreguidelines/use-enum-class.cpp  | 62 ++++++++++++++++++++++
 8 files changed, 190 insertions(+)
 create mode 100644 clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.cpp
 create mode 100644 clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.h
 create mode 100644 clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/use-enum-class.rst
 create mode 100644 clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/use-enum-class.cpp

diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
index b023f76a2543..2fb4d7f1d734 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CMakeLists.txt
@@ -33,6 +33,7 @@ add_clang_library(clangTidyCppCoreGuidelinesModule STATIC
   RvalueReferenceParamNotMovedCheck.cpp
   SlicingCheck.cpp
   SpecialMemberFunctionsCheck.cpp
+  UseEnumClassCheck.cpp
   VirtualClassDestructorCheck.cpp
 
   LINK_LIBS
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
index 4dd9b0904f07..4b3b7bf963fd 100644
--- a/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/CppCoreGuidelinesTidyModule.cpp
@@ -48,6 +48,7 @@
 #include "RvalueReferenceParamNotMovedCheck.h"
 #include "SlicingCheck.h"
 #include "SpecialMemberFunctionsCheck.h"
+#include "UseEnumClassCheck.h"
 #include "VirtualClassDestructorCheck.h"
 
 namespace clang::tidy {
@@ -131,6 +132,8 @@ public:
     CheckFactories.registerCheck<SlicingCheck>("cppcoreguidelines-slicing");
     CheckFactories.registerCheck<modernize::UseDefaultMemberInitCheck>(
         "cppcoreguidelines-use-default-member-init");
+    CheckFactories.registerCheck<UseEnumClassCheck>(
+        "cppcoreguidelines-use-enum-class");
     CheckFactories.registerCheck<misc::UnconventionalAssignOperatorCheck>(
         "cppcoreguidelines-c-copy-assignment-signature");
     CheckFactories.registerCheck<VirtualClassDestructorCheck>(
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.cpp b/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.cpp
new file mode 100644
index 000000000000..ec7d9237afa3
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.cpp
@@ -0,0 +1,42 @@
+//===--- UseEnumClassCheck.cpp - clang-tidy -------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "UseEnumClassCheck.h"
+#include "clang/ASTMatchers/ASTMatchFinder.h"
+
+using namespace clang::ast_matchers;
+
+namespace clang::tidy::cppcoreguidelines {
+
+UseEnumClassCheck::UseEnumClassCheck(StringRef Name, ClangTidyContext *Context)
+    : ClangTidyCheck(Name, Context),
+      IgnoreUnscopedEnumsInClasses(
+          Options.get("IgnoreUnscopedEnumsInClasses", false)) {}
+
+void UseEnumClassCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) {
+  Options.store(Opts, "IgnoreUnscopedEnumsInClasses",
+                IgnoreUnscopedEnumsInClasses);
+}
+
+void UseEnumClassCheck::registerMatchers(MatchFinder *Finder) {
+  auto EnumDecl =
+      IgnoreUnscopedEnumsInClasses
+          ? enumDecl(unless(isScoped()), unless(hasParent(recordDecl())))
+          : enumDecl(unless(isScoped()));
+  Finder->addMatcher(EnumDecl.bind("unscoped_enum"), this);
+}
+
+void UseEnumClassCheck::check(const MatchFinder::MatchResult &Result) {
+  const auto *UnscopedEnum = Result.Nodes.getNodeAs<EnumDecl>("unscoped_enum");
+
+  diag(UnscopedEnum->getLocation(),
+       "enum %0 is unscoped, use 'enum class' instead")
+      << UnscopedEnum;
+}
+
+} // namespace clang::tidy::cppcoreguidelines
diff --git a/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.h b/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.h
new file mode 100644
index 000000000000..dfa4b7e3fda6
--- /dev/null
+++ b/clang-tools-extra/clang-tidy/cppcoreguidelines/UseEnumClassCheck.h
@@ -0,0 +1,40 @@
+//===--- UseEnumClassCheck.h - clang-tidy -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_USEENUMCLASSCHECK_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_USEENUMCLASSCHECK_H
+
+#include "../ClangTidyCheck.h"
+
+namespace clang::tidy::cppcoreguidelines {
+
+/// Finds unscoped (non-class) enum declarations and suggests using enum class
+/// instead.
+///
+/// For the user-facing documentation see:
+/// http://clang.llvm.org/extra/clang-tidy/checks/cppcoreguidelines/use-enum-class.html
+class UseEnumClassCheck : public ClangTidyCheck {
+public:
+  UseEnumClassCheck(StringRef Name, ClangTidyContext *Context);
+  void storeOptions(ClangTidyOptions::OptionMap &Opts) override;
+  void registerMatchers(ast_matchers::MatchFinder *Finder) override;
+  void check(const ast_matchers::MatchFinder::MatchResult &Result) override;
+  bool isLanguageVersionSupported(const LangOptions &LangOpts) const override {
+    return LangOpts.CPlusPlus11;
+  }
+  std::optional<TraversalKind> getCheckTraversalKind() const override {
+    return TraversalKind::TK_IgnoreUnlessSpelledInSource;
+  }
+
+private:
+  const bool IgnoreUnscopedEnumsInClasses;
+};
+
+} // namespace clang::tidy::cppcoreguidelines
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_CPPCOREGUIDELINES_USEENUMCLASSCHECK_H
diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst
index 3c1ca2f92904..7c0c534dbc73 100644
--- a/clang-tools-extra/docs/ReleaseNotes.rst
+++ b/clang-tools-extra/docs/ReleaseNotes.rst
@@ -136,6 +136,12 @@ New checks
   Finds unintended character output from ``unsigned char`` and ``signed char``
   to an ``ostream``.
 
+- New :doc:`cppcoreguidelines-use-enum-class
+  <clang-tidy/checks/cppcoreguidelines/use-enum-class>` check.
+
+  Finds unscoped (non-class) ``enum`` declarations and suggests using
+  ``enum class`` instead.
+
 - New :doc:`portability-avoid-pragma-once
   <clang-tidy/checks/portability/avoid-pragma-once>` check.
 
diff --git a/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/use-enum-class.rst b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/use-enum-class.rst
new file mode 100644
index 000000000000..9e9f4c99dc24
--- /dev/null
+++ b/clang-tools-extra/docs/clang-tidy/checks/cppcoreguidelines/use-enum-class.rst
@@ -0,0 +1,35 @@
+.. title:: clang-tidy - cppcoreguidelines-use-enum-class
+
+cppcoreguidelines-use-enum-class
+================================
+
+Finds unscoped (non-class) ``enum`` declarations and suggests using
+``enum class`` instead.
+
+This check implements `Enum.3
+<https://isocpp.github.io/CppCoreGuidelines/CppCoreGuidelines#Renum-class>`_
+from the C++ Core Guidelines."
+
+Example:
+
+.. code-block:: c++
+
+  enum E {};        // use "enum class E {};" instead
+  enum class E {};  // OK
+
+  struct S {
+      enum E {};    // use "enum class E {};" instead
+                    // OK with option IgnoreUnscopedEnumsInClasses
+  };
+
+  namespace N {
+      enum E {};    // use "enum class E {};" instead
+  }
+
+Options
+-------
+
+.. option:: IgnoreUnscopedEnumsInClasses
+
+   When `true`, ignores unscoped ``enum`` declarations in classes.
+   Default is `false`.
diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst
index 5a79d61b1fd7..ccb78ee45e9c 100644
--- a/clang-tools-extra/docs/clang-tidy/checks/list.rst
+++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst
@@ -212,6 +212,7 @@ Clang-Tidy Checks
    :doc:`cppcoreguidelines-rvalue-reference-param-not-moved <cppcoreguidelines/rvalue-reference-param-not-moved>`,
    :doc:`cppcoreguidelines-slicing <cppcoreguidelines/slicing>`,
    :doc:`cppcoreguidelines-special-member-functions <cppcoreguidelines/special-member-functions>`,
+   :doc:`cppcoreguidelines-use-enum-class <cppcoreguidelines/use-enum-class>`,
    :doc:`cppcoreguidelines-virtual-class-destructor <cppcoreguidelines/virtual-class-destructor>`, "Yes"
    :doc:`darwin-avoid-spinlock <darwin/avoid-spinlock>`,
    :doc:`darwin-dispatch-once-nonstatic <darwin/dispatch-once-nonstatic>`, "Yes"
diff --git a/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/use-enum-class.cpp b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/use-enum-class.cpp
new file mode 100644
index 000000000000..f53d787f80ef
--- /dev/null
+++ b/clang-tools-extra/test/clang-tidy/checkers/cppcoreguidelines/use-enum-class.cpp
@@ -0,0 +1,62 @@
+// RUN: %check_clang_tidy -std=c++11-or-later -check-suffix=ALL,DEFAULT %s \
+// RUN: cppcoreguidelines-use-enum-class %t --
+
+// RUN: %check_clang_tidy -std=c++11-or-later -check-suffix=ALL %s \
+// RUN: cppcoreguidelines-use-enum-class %t -- \
+// RUN: -config="{CheckOptions: { \
+// RUN: cppcoreguidelines-use-enum-class.IgnoreUnscopedEnumsInClasses: true \
+// RUN: }}" --
+
+enum E {};
+// CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: enum 'E' is unscoped, use 'enum class' instead
+
+enum class EC {};
+
+enum struct ES {};
+
+struct S {
+  enum E {};
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:8: warning: enum 'E' is unscoped, use 'enum class' instead
+  enum class EC {};
+};
+
+class C {
+  enum E {};
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:8: warning: enum 'E' is unscoped, use 'enum class' instead
+  enum class EC {};
+};
+
+template<class T>
+class TC {
+  enum E {};
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:8: warning: enum 'E' is unscoped, use 'enum class' instead
+  enum class EC {};
+};
+
+union U {
+  enum E {};
+  // CHECK-MESSAGES-DEFAULT: :[[@LINE-1]]:8: warning: enum 'E' is unscoped, use 'enum class' instead
+  enum class EC {};
+};
+
+namespace {
+enum E {};
+// CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: enum 'E' is unscoped, use 'enum class' instead
+enum class EC {};
+} // namespace
+
+namespace N {
+enum E {};
+// CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: enum 'E' is unscoped, use 'enum class' instead
+enum class EC {};
+} // namespace N
+
+template<enum ::EC>
+static void foo();
+
+enum ForwardE : int;
+// CHECK-MESSAGES-ALL: :[[@LINE-1]]:6: warning: enum 'ForwardE' is unscoped, use 'enum class' instead
+
+enum class ForwardEC : int;
+
+enum struct ForwardES : int;
-- 
cgit v1.2.3


From 43e1a5a411d972fe06a1afb86ffd5ba21fd2a376 Mon Sep 17 00:00:00 2001
From: Frank Schlimbach <frank.schlimbach@intel.com>
Date: Wed, 18 Jun 2025 11:06:48 +0200
Subject: [mlir][mesh] adding option for traversal order in sharding
 propagation (#144079)

The traversal order in sharding propagation was hard-coded. This PR
provides options to the pass to select a suitable order
- forward-only
- backward-only
- forward-backward
- backward-forward

Default is the previous behavior (backward-forward).
---
 mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h        |  3 --
 mlir/include/mlir/Dialect/Mesh/Transforms/Passes.h | 12 ++++++
 .../include/mlir/Dialect/Mesh/Transforms/Passes.td | 15 +++++++
 mlir/lib/Dialect/Mesh/IR/MeshOps.cpp               | 27 ++++++------
 .../Mesh/Transforms/ShardingPropagation.cpp        | 42 +++++++++++++------
 .../Mesh/backward-sharding-propagation.mlir        | 26 ++++++++++++
 .../forward-backward-sharding-propagation.mlir     | 27 ++++++++++++
 .../Dialect/Mesh/forward-sharding-propagation.mlir | 49 ++++++++++++++++++++++
 8 files changed, 173 insertions(+), 28 deletions(-)
 create mode 100644 mlir/test/Dialect/Mesh/backward-sharding-propagation.mlir
 create mode 100644 mlir/test/Dialect/Mesh/forward-backward-sharding-propagation.mlir
 create mode 100644 mlir/test/Dialect/Mesh/forward-sharding-propagation.mlir

diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
index 32c2eca2cefa..3878505f8f93 100644
--- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
+++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.h
@@ -206,9 +206,6 @@ Type shardType(Type type, MeshOp mesh, MeshSharding sharding);
 // Use newShardOp if it is not null. Otherwise create a new one.
 // May insert resharding if required.
 // Potentially updates newShardOp.
-void maybeInsertTargetShardingAnnotation(MeshSharding sharding,
-                                         OpOperand &operand, OpBuilder &builder,
-                                         ShardOp &newShardOp);
 void maybeInsertTargetShardingAnnotation(MeshSharding sharding, OpResult result,
                                          OpBuilder &builder);
 void maybeInsertSourceShardingAnnotation(MeshSharding sharding,
diff --git a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.h b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.h
index 83399d10beaa..a2424d43a8ba 100644
--- a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.h
@@ -19,6 +19,18 @@ class FuncOp;
 
 namespace mesh {
 
+/// This enum controls the traversal order for the sharding propagation.
+enum class TraversalOrder {
+  /// Forward traversal.
+  Forward,
+  /// Backward traversal.
+  Backward,
+  /// Forward then backward traversal.
+  ForwardBackward,
+  /// Backward then forward traversal.
+  BackwardForward
+};
+
 //===----------------------------------------------------------------------===//
 // Passes
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
index 06ebf151e7d6..11ec7e78cd5e 100644
--- a/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Mesh/Transforms/Passes.td
@@ -24,6 +24,21 @@ def ShardingPropagation : InterfacePass<"sharding-propagation", "mlir::FunctionO
     operation, and the operations themselves are added with sharding option
     attributes.
   }];
+  let options = [
+    Option<"traversal", "traversal",
+           "mlir::mesh::TraversalOrder", /*default=*/"mlir::mesh::TraversalOrder::BackwardForward",
+           "Traversal order to use for sharding propagation:",
+            [{::llvm::cl::values(
+              clEnumValN(mlir::mesh::TraversalOrder::Forward, "forward",
+              "Forward only traversal."),
+              clEnumValN(mlir::mesh::TraversalOrder::Backward, "backward",
+              "backward only traversal."),
+              clEnumValN(mlir::mesh::TraversalOrder::ForwardBackward, "forward-backward",
+              "forward-backward traversal."),
+              clEnumValN(mlir::mesh::TraversalOrder::BackwardForward, "backward-forward",
+              "backward-forward traversal.")
+            )}]>,
+  ];
   let dependentDialects = [
     "mesh::MeshDialect"
   ];
diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
index 304cb55a3508..a2c2d1a7470c 100644
--- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
+++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp
@@ -275,13 +275,12 @@ Type mesh::shardType(Type type, MeshOp mesh, MeshSharding sharding) {
   return type;
 }
 
-void mlir::mesh::maybeInsertTargetShardingAnnotation(MeshSharding sharding,
-                                                     OpOperand &operand,
-                                                     OpBuilder &builder,
-                                                     ShardOp &newShardOp) {
+static void maybeInsertTargetShardingAnnotationImpl(MeshSharding sharding,
+                                                    Value &operandValue,
+                                                    Operation *operandOp,
+                                                    OpBuilder &builder,
+                                                    ShardOp &newShardOp) {
   OpBuilder::InsertionGuard insertionGuard(builder);
-  Value operandValue = operand.get();
-  Operation *operandOp = operand.getOwner();
   builder.setInsertionPointAfterValue(operandValue);
   ShardOp shardOp = dyn_cast<ShardOp>(operandOp);
   if (shardOp && sharding == shardOp.getSharding() &&
@@ -300,9 +299,8 @@ void mlir::mesh::maybeInsertTargetShardingAnnotation(MeshSharding sharding,
         builder.create<ShardOp>(operandValue.getLoc(), operandValue, shardingOp,
                                 /*annotate_for_users*/ false);
   }
-  IRRewriter rewriter(builder);
-  rewriter.replaceUsesWithIf(
-      operandValue, newShardOp, [operandOp, operandValue](OpOperand &use) {
+  operandValue.replaceUsesWithIf(
+      newShardOp, [operandOp, operandValue](OpOperand &use) {
         return use.getOwner() == operandOp && use.get() == operandValue;
       });
 
@@ -313,15 +311,20 @@ void mlir::mesh::maybeInsertTargetShardingAnnotation(MeshSharding sharding,
   auto newShardOp2 = builder.create<ShardOp>(operandValue.getLoc(), newShardOp,
                                              newShardOp.getSharding(),
                                              /*annotate_for_users*/ true);
-  rewriter.replaceAllUsesExcept(newShardOp, newShardOp2, newShardOp2);
+  newShardOp.getResult().replaceAllUsesExcept(newShardOp2, newShardOp2);
 }
 
 void mlir::mesh::maybeInsertTargetShardingAnnotation(MeshSharding sharding,
                                                      OpResult result,
                                                      OpBuilder &builder) {
   ShardOp newShardOp;
-  for (auto &use : llvm::make_early_inc_range(result.getUses())) {
-    maybeInsertTargetShardingAnnotation(sharding, use, builder, newShardOp);
+  SmallVector<std::pair<Value, Operation *>> uses;
+  for (auto &use : result.getUses()) {
+    uses.emplace_back(use.get(), use.getOwner());
+  }
+  for (auto &[operandValue, operandOp] : uses) {
+    maybeInsertTargetShardingAnnotationImpl(sharding, operandValue, operandOp,
+                                            builder, newShardOp);
   }
 }
 
diff --git a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
index 4452dd65fce9..6751fafaf177 100644
--- a/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
+++ b/mlir/lib/Dialect/Mesh/Transforms/ShardingPropagation.cpp
@@ -362,6 +362,9 @@ static LogicalResult visitOp(Operation *op, OpBuilder &builder) {
 //===----------------------------------------------------------------------===//
 struct ShardingPropagation
     : public mesh::impl::ShardingPropagationBase<ShardingPropagation> {
+
+  using ShardingPropagationBase<ShardingPropagation>::ShardingPropagationBase;
+
   void runOnOperation() override {
     FunctionOpInterface funcOp = getOperation();
     MLIRContext *ctx = funcOp.getContext();
@@ -382,18 +385,31 @@ struct ShardingPropagation
             shardingOp.printLoopTypesAndIndexingMaps(llvm::dbgs());
         });
 
-    // 1. propagate in reversed order
-    for (Operation &op : llvm::make_early_inc_range(llvm::reverse(block)))
-      if (failed(visitOp(&op, builder)))
-        return signalPassFailure();
-
-    LLVM_DEBUG(DBGS() << "After reversed order propagation:\n"
-                      << funcOp << "\n");
-    LLVM_DEBUG(assert(succeeded(mlir::verify(funcOp))));
-
-    // 2. propagate in original order
-    for (Operation &op : llvm::make_early_inc_range(block))
-      if (failed(visitOp(&op, builder)))
-        return signalPassFailure();
+    auto traverse = [&](auto &&range, OpBuilder &builder,
+                        const char *order) -> bool {
+      for (Operation &op : range) {
+        if (failed(visitOp(&op, builder))) {
+          signalPassFailure();
+          return true;
+        }
+      }
+      LLVM_DEBUG(DBGS() << "After " << order << " order propagation:\n"
+                        << funcOp << "\n");
+      LLVM_DEBUG(assert(succeeded(mlir::verify(funcOp))));
+      return false;
+    };
+
+    // 1. Propagate in reversed order.
+    if (traversal == TraversalOrder::Backward ||
+        traversal == TraversalOrder::BackwardForward)
+      traverse(llvm::reverse(block), builder, "backward");
+
+    // 2. Propagate in original order.
+    if (traversal != TraversalOrder::Backward)
+      traverse(block, builder, "forward");
+
+    // 3. Propagate in backward order if needed.
+    if (traversal == TraversalOrder::ForwardBackward)
+      traverse(llvm::reverse(block), builder, "backward");
   }
 };
diff --git a/mlir/test/Dialect/Mesh/backward-sharding-propagation.mlir b/mlir/test/Dialect/Mesh/backward-sharding-propagation.mlir
new file mode 100644
index 000000000000..4223d01d6511
--- /dev/null
+++ b/mlir/test/Dialect/Mesh/backward-sharding-propagation.mlir
@@ -0,0 +1,26 @@
+// RUN: mlir-opt --pass-pipeline="builtin.module(func.func(sharding-propagation{traversal=backward}))" %s | FileCheck %s
+
+#map = affine_map<(d0, d1) -> (d0, d1)>
+module {
+  mesh.mesh @mesh(shape = 1) {sym_visibility = "private"}
+  func.func @test_forward() -> tensor<6x6xi32> {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: tensor.empty()
+    %0 = tensor.empty() : tensor<6x6xi32>
+    %sharding = mesh.sharding @mesh split_axes = [[0]] : !mesh.sharding
+    // CHECK-COUNT-2: mesh.shard
+    %sharding_annotated = mesh.shard %0 to %sharding : tensor<6x6xi32>
+    %1 = linalg.fill ins(%c1_i32 : i32) outs(%sharding_annotated : tensor<6x6xi32>) -> tensor<6x6xi32>
+    // CHECK: tensor.empty()
+    // CHECK-NOT: mesh.shard @
+    %2 = tensor.empty() : tensor<6x6xi32>
+    %3 = linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%1, %1
+        : tensor<6x6xi32>, tensor<6x6xi32>) outs(%2 : tensor<6x6xi32>) {
+    ^bb0(%in: i32, %in_2: i32, %out: i32):
+      %9 = arith.addi %in, %in_2 : i32
+      linalg.yield %9 : i32
+    } -> tensor<6x6xi32>
+    // CHECK: return
+    return %3 : tensor<6x6xi32>
+  }
+}
diff --git a/mlir/test/Dialect/Mesh/forward-backward-sharding-propagation.mlir b/mlir/test/Dialect/Mesh/forward-backward-sharding-propagation.mlir
new file mode 100644
index 000000000000..dd2eee2f7def
--- /dev/null
+++ b/mlir/test/Dialect/Mesh/forward-backward-sharding-propagation.mlir
@@ -0,0 +1,27 @@
+// RUN: mlir-opt --pass-pipeline="builtin.module(func.func(sharding-propagation{traversal=forward-backward}))" %s | FileCheck %s
+
+#map = affine_map<(d0, d1) -> (d0, d1)>
+module {
+  mesh.mesh @mesh(shape = 1) {sym_visibility = "private"}
+  func.func @test_forward() -> tensor<6x6xi32> {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: tensor.empty()
+    %0 = tensor.empty() : tensor<6x6xi32>
+    // CHECK-COUNT-3: mesh.sharding @mesh split_axes = {{\[\[0}}]]
+    %sharding_row = mesh.sharding @mesh split_axes = [[0]] : !mesh.sharding
+    %annotated_row = mesh.shard %0 to %sharding_row : tensor<6x6xi32>
+    %1 = linalg.fill ins(%c1_i32 : i32) outs(%annotated_row : tensor<6x6xi32>) -> tensor<6x6xi32>
+    %2 = tensor.empty() : tensor<6x6xi32>
+    // CHECK-COUNT-4: mesh.sharding @mesh split_axes = {{\[\[1}}]]
+    %3 = linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%2, %1
+        : tensor<6x6xi32>, tensor<6x6xi32>) outs(%2 : tensor<6x6xi32>) {
+    ^bb0(%in: i32, %in_2: i32, %out: i32):
+      %9 = arith.addi %in, %in_2 : i32
+      linalg.yield %9 : i32
+    } -> tensor<6x6xi32>
+    %sharding_col = mesh.sharding @mesh split_axes = [[1]] : !mesh.sharding
+    %annotated_col = mesh.shard %3 to %sharding_col : tensor<6x6xi32>
+    // CHECK: return
+    return %annotated_col : tensor<6x6xi32>
+  }
+}
diff --git a/mlir/test/Dialect/Mesh/forward-sharding-propagation.mlir b/mlir/test/Dialect/Mesh/forward-sharding-propagation.mlir
new file mode 100644
index 000000000000..98e9931b8de9
--- /dev/null
+++ b/mlir/test/Dialect/Mesh/forward-sharding-propagation.mlir
@@ -0,0 +1,49 @@
+// RUN: mlir-opt --pass-pipeline="builtin.module(func.func(sharding-propagation{traversal=forward}))" %s | FileCheck %s
+
+#map = affine_map<(d0, d1) -> (d0, d1)>
+module attributes {dlti.map = #dlti.map<"MPI:Implementation" = "mpich", "MPI:comm_world_rank" = 0 : i32>} {
+  mesh.mesh @mesh(shape = 1) {sym_visibility = "private"}
+  func.func @test_forward() -> (tensor<6x6xi32>, tensor<6x6xi32>, tensor<i32>) attributes {llvm.emit_c_interface} {
+    %c1_i32 = arith.constant 1 : i32
+    // CHECK: [[v3:%.*]] = tensor.empty() : tensor<6x6xi32>
+    %0 = tensor.empty() : tensor<6x6xi32>
+    // CHECK: [[v1:%.*]] = linalg.fill ins
+    // CHECK: [[vsharding_0:%.*]] = mesh.sharding @mesh split_axes = {{\[\[}}0]] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_1:%.*]] = mesh.shard [[v1]] to [[vsharding_0]] : tensor<6x6xi32>
+    %1 = linalg.fill ins(%c1_i32 : i32) outs(%0 : tensor<6x6xi32>) -> tensor<6x6xi32>
+    %sharding = mesh.sharding @mesh split_axes = [[0]] : !mesh.sharding
+    %sharding_annotated = mesh.shard %1 to %sharding : tensor<6x6xi32>
+    // CHECK: [[v2:%.*]] = tensor.empty() : tensor<6x6xi32>
+    // CHECK: [[vsharding_2:%.*]] = mesh.sharding @mesh split_axes = {{\[\[}}0]] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_3:%.*]] = mesh.shard [[vsharding_annotated_1]] to [[vsharding_2]] annotate_for_users : tensor<6x6xi32>
+    %3 = tensor.empty() : tensor<6x6xi32>
+    // CHECK: [[vsharding_4:%.*]] = mesh.sharding @mesh split_axes = {{\[\[}}0]] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_5:%.*]] = mesh.shard [[v2]] to [[vsharding_4]] annotate_for_users : tensor<6x6xi32>
+    // CHECK: [[v3:%.*]] = linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]}
+    // CHECK-SAME: ins([[vsharding_annotated_3]], [[vsharding_annotated_3]] : tensor<6x6xi32>, tensor<6x6xi32>) outs([[vsharding_annotated_5]] : tensor<6x6xi32>) {
+    // CHECK: [[vsharding_6:%.*]] = mesh.sharding @mesh split_axes = {{\[\[}}0]] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_7:%.*]] = mesh.shard [[v3]] to [[vsharding_6]] : tensor<6x6xi32>
+    %4 = linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%sharding_annotated, %sharding_annotated
+        : tensor<6x6xi32>, tensor<6x6xi32>) outs(%3 : tensor<6x6xi32>) {
+    ^bb0(%in: i32, %in_2: i32, %out: i32):
+      %9 = arith.addi %in, %in_2 : i32
+      linalg.yield %9 : i32
+    } -> tensor<6x6xi32>
+    %c0_i32 = arith.constant 0 : i32
+    %6 = tensor.empty() : tensor<i32>
+    %7 = linalg.fill ins(%c0_i32 : i32) outs(%6 : tensor<i32>) -> tensor<i32>
+    // CHECK: [[vreduced:%.*]] = linalg.reduce ins
+    // CHECK: [[vsharding_12:%.*]] = mesh.sharding @mesh split_axes = [] partial =  sum [0] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_13:%.*]] = mesh.shard [[vreduced]] to [[vsharding_12]] : tensor<i32>
+    %reduced = linalg.reduce ins(%4 : tensor<6x6xi32>) outs(%7 : tensor<i32>) dimensions = [0, 1] 
+      (%in: i32, %init: i32) {
+        %9 = arith.addi %in, %init : i32
+        linalg.yield %9 : i32
+      }
+    // CHECK: [[vsharding_14:%.*]] = mesh.sharding @mesh split_axes = {{\[\[}}]] : !mesh.sharding
+    %sharding_0 = mesh.sharding @mesh split_axes = [[]] : !mesh.sharding
+    // CHECK: [[vsharding_annotated_15:%.*]] = mesh.shard [[vsharding_annotated_13]] to [[vsharding_14]] annotate_for_users : tensor<i32>
+    %sharding_annotated_1 = mesh.shard %reduced to %sharding_0 annotate_for_users : tensor<i32>
+    return %sharding_annotated, %4, %sharding_annotated_1 : tensor<6x6xi32>, tensor<6x6xi32>, tensor<i32>
+  }
+}
-- 
cgit v1.2.3


From 355725a25e6be38d7a97cab9e206d2a16a1bd849 Mon Sep 17 00:00:00 2001
From: Kunqiu Chen <camsyn@foxmail.com>
Date: Wed, 18 Jun 2025 17:09:32 +0800
Subject: [TSan] Fix missing inst cleanup (#144067)

Commit 44e875ad5b2ce26826dd53f9e7d1a71436c86212 introduced a change that
replaces `ReplaceInstWithInst` with `Instruction::replaceAllUsesWith`,
without subsequent instruction cleanup.

This results in TSan leaving behind useless `load atomic` instructions
after 'replacing' them.

This commit adds cleanup back, consistent with the context.
---
 llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp         | 1 +
 llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll | 3 ---
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index 8ae6f7745a9e..5485998164f1 100644
--- a/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -728,6 +728,7 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I, const DataLayout &DL) {
     Value *C = IRB.CreateCall(TsanAtomicLoad[Idx], Args);
     Value *Cast = IRB.CreateBitOrPointerCast(C, OrigTy);
     I->replaceAllUsesWith(Cast);
+    I->eraseFromParent();
   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
     Value *Addr = SI->getPointerOperand();
     int Idx =
diff --git a/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll b/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
index 8bcabaecf0fd..015ee2fe711e 100644
--- a/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
+++ b/llvm/test/Instrumentation/ThreadSanitizer/atomic-non-integer.ll
@@ -10,7 +10,6 @@ define float @load_float(ptr %fptr) {
 ; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @__tsan_atomic32_load(ptr [[FPTR]], i32 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i32 [[TMP2]] to float
-; CHECK-NEXT:    [[V:%.*]] = load atomic float, ptr [[FPTR]] unordered, align 4
 ; CHECK-NEXT:    call void @__tsan_func_exit()
 ; CHECK-NEXT:    ret float [[TMP3]]
 ;
@@ -25,7 +24,6 @@ define double @load_double(ptr %fptr) {
 ; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @__tsan_atomic64_load(ptr [[FPTR]], i32 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i64 [[TMP2]] to double
-; CHECK-NEXT:    [[V:%.*]] = load atomic double, ptr [[FPTR]] unordered, align 8
 ; CHECK-NEXT:    call void @__tsan_func_exit()
 ; CHECK-NEXT:    ret double [[TMP3]]
 ;
@@ -40,7 +38,6 @@ define fp128 @load_fp128(ptr %fptr) {
 ; CHECK-NEXT:    call void @__tsan_func_entry(ptr [[TMP1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i128 @__tsan_atomic128_load(ptr [[FPTR]], i32 0)
 ; CHECK-NEXT:    [[TMP3:%.*]] = bitcast i128 [[TMP2]] to fp128
-; CHECK-NEXT:    [[V:%.*]] = load atomic fp128, ptr [[FPTR]] unordered, align 16
 ; CHECK-NEXT:    call void @__tsan_func_exit()
 ; CHECK-NEXT:    ret fp128 [[TMP3]]
 ;
-- 
cgit v1.2.3


From 8e157fdbb7b4af9f67b139a9f05feaa9b338d3f5 Mon Sep 17 00:00:00 2001
From: Sirui Mu <msrlancern@gmail.com>
Date: Wed, 18 Jun 2025 17:10:29 +0800
Subject: [CIR] Add support for __builtin_assume (#144376)

This patch adds support for the `__builtin_assume` builtin function.
---
 clang/include/clang/CIR/Dialect/IR/CIROps.td       | 22 +++++++++++++
 clang/include/clang/CIR/MissingFeatures.h          |  3 ++
 clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp            | 38 ++++++++++++++++++++++
 clang/lib/CIR/CodeGen/CIRGenFunction.h             |  4 +++
 .../lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp  |  9 +++++
 clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h  | 10 ++++++
 clang/test/CIR/CodeGen/builtin_call.cpp            | 16 +++++++++
 7 files changed, 102 insertions(+)

diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 8dd1f0ce361d..4655cebc82ee 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -2387,4 +2387,26 @@ def ComplexCreateOp : CIR_Op<"complex.create", [Pure, SameTypeOperands]> {
   let hasFolder = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// Assume Operations
+//===----------------------------------------------------------------------===//
+
+def AssumeOp : CIR_Op<"assume"> {
+  let summary = "Tell the optimizer that a boolean value is true";
+  let description = [{
+    The `cir.assume` operation takes a single boolean prediate as its only
+    argument and does not have any results. The operation tells the optimizer
+    that the predicate is always true.
+
+    This operation corresponds to the `__assume` and the `__builtin_assume`
+    builtin functions.
+  }];
+
+  let arguments = (ins CIR_BoolType:$predicate);
+
+  let assemblyFormat = [{
+    $predicate `:` type($predicate) attr-dict
+  }];
+}
+
 #endif // CLANG_CIR_DIALECT_IR_CIROPS_TD
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 3dc28e6f2e5b..3d120903dea1 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -237,6 +237,9 @@ struct MissingFeatures {
   static bool lowerAggregateLoadStore() { return false; }
   static bool dataLayoutTypeAllocSize() { return false; }
   static bool asmLabelAttr() { return false; }
+  static bool builtinCall() { return false; }
+  static bool builtinCallF128() { return false; }
+  static bool builtinCallMathErrno() { return false; }
 
   // Missing types
   static bool dataMemberType() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 19fac00ab873..83825f0835a1 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "CIRGenCall.h"
+#include "CIRGenConstantEmitter.h"
 #include "CIRGenFunction.h"
 #include "CIRGenModule.h"
 #include "CIRGenValue.h"
@@ -66,6 +67,32 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
     return emitLibraryCall(*this, fd, e,
                            cgm.getBuiltinLibFunction(fd, builtinID));
 
+  assert(!cir::MissingFeatures::builtinCallF128());
+
+  // If the builtin has been declared explicitly with an assembler label,
+  // disable the specialized emitting below. Ideally we should communicate the
+  // rename in IR, or at least avoid generating the intrinsic calls that are
+  // likely to get lowered to the renamed library functions.
+  unsigned builtinIDIfNoAsmLabel = fd->hasAttr<AsmLabelAttr>() ? 0 : builtinID;
+
+  assert(!cir::MissingFeatures::builtinCallMathErrno());
+  assert(!cir::MissingFeatures::builtinCall());
+
+  switch (builtinIDIfNoAsmLabel) {
+  default:
+    break;
+
+  case Builtin::BI__assume:
+  case Builtin::BI__builtin_assume: {
+    if (e->getArg(0)->HasSideEffects(getContext()))
+      return RValue::get(nullptr);
+
+    mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0));
+    builder.create<cir::AssumeOp>(getLoc(e->getExprLoc()), argValue);
+    return RValue::get(nullptr);
+  }
+  }
+
   cgm.errorNYI(e->getSourceRange(), "unimplemented builtin call");
   return getUndefRValue(e->getType());
 }
@@ -88,3 +115,14 @@ cir::FuncOp CIRGenModule::getBuiltinLibFunction(const FunctionDecl *fd,
   mlir::Type type = convertType(fd->getType());
   return getOrCreateCIRFunction(name, type, d, /*forVTable=*/false);
 }
+
+mlir::Value CIRGenFunction::emitCheckedArgForAssume(const Expr *e) {
+  mlir::Value argValue = evaluateExprAsBool(e);
+  if (!sanOpts.has(SanitizerKind::Builtin))
+    return argValue;
+
+  assert(!cir::MissingFeatures::sanitizers());
+  cgm.errorNYI(e->getSourceRange(),
+               "emitCheckedArgForAssume: sanitizers are NYI");
+  return {};
+}
diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index de6ef2a69faf..6c490a72b2e9 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -772,6 +772,10 @@ public:
 
   LValue emitCastLValue(const CastExpr *e);
 
+  /// Emits an argument for a call to a `__builtin_assume`. If the builtin
+  /// sanitizer is enabled, a runtime check is also emitted.
+  mlir::Value emitCheckedArgForAssume(const Expr *e);
+
   LValue emitCompoundAssignmentLValue(const clang::CompoundAssignOperator *e);
 
   void emitConstructorBody(FunctionArgList &args);
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index 6a4e4e4a7df3..a96501ab2c38 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -407,6 +407,14 @@ struct ConvertCIRToLLVMPass
   StringRef getArgument() const override { return "cir-flat-to-llvm"; }
 };
 
+mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
+    cir::AssumeOp op, OpAdaptor adaptor,
+    mlir::ConversionPatternRewriter &rewriter) const {
+  auto cond = adaptor.getPredicate();
+  rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);
+  return mlir::success();
+}
+
 mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
     cir::BrCondOp brOp, OpAdaptor adaptor,
     mlir::ConversionPatternRewriter &rewriter) const {
@@ -1811,6 +1819,7 @@ void ConvertCIRToLLVMPass::runOnOperation() {
                                              dl);
   patterns.add<
       // clang-format off
+               CIRToLLVMAssumeOpLowering,
                CIRToLLVMBaseClassAddrOpLowering,
                CIRToLLVMBinOpLowering,
                CIRToLLVMBrCondOpLowering,
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index a80981806354..a80c66ac1abf 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -29,6 +29,16 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, mlir::Attribute attr,
 
 mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage);
 
+class CIRToLLVMAssumeOpLowering
+    : public mlir::OpConversionPattern<cir::AssumeOp> {
+public:
+  using mlir::OpConversionPattern<cir::AssumeOp>::OpConversionPattern;
+
+  mlir::LogicalResult
+  matchAndRewrite(cir::AssumeOp op, OpAdaptor,
+                  mlir::ConversionPatternRewriter &) const override;
+};
+
 class CIRToLLVMBrCondOpLowering
     : public mlir::OpConversionPattern<cir::BrCondOp> {
 public:
diff --git a/clang/test/CIR/CodeGen/builtin_call.cpp b/clang/test/CIR/CodeGen/builtin_call.cpp
index 322c13c8f081..0a2226a2cc59 100644
--- a/clang/test/CIR/CodeGen/builtin_call.cpp
+++ b/clang/test/CIR/CodeGen/builtin_call.cpp
@@ -94,3 +94,19 @@ void library_builtins() {
 // OGCG: define dso_local void @_Z16library_builtinsv()
 // OGCG: call i32 (ptr, ...) @printf(ptr noundef null)
 // OGCG: call void @abort()
+
+void assume(bool arg) {
+  __builtin_assume(arg);
+}
+
+// CIR: cir.func @_Z6assumeb
+// CIR:   cir.assume %{{.+}} : !cir.bool
+// CIR: }
+
+// LLVM: define void @_Z6assumeb
+// LLVM:   call void @llvm.assume(i1 %{{.+}})
+// LLVM: }
+
+// OGCG: define {{.*}}void @_Z6assumeb
+// OGCG:   call void @llvm.assume(i1 %{{.+}})
+// OGCG: }
-- 
cgit v1.2.3


From fe42d34274cac79794637bf2f69f85537dde8b74 Mon Sep 17 00:00:00 2001
From: Ying Yi <ying.yi@sony.com>
Date: Wed, 18 Jun 2025 10:13:46 +0100
Subject: [clang][headers]Remove unnecessary guard of !defined(__SCE__).
 (#144522)

Sony PlayStation now supports C++20, and we wish to change the default
C++ mode to C++20 sometime in the future. As such, the !defined(__SCE__)
guards are redundant and we want to remove them. This in turn makes the
entire guard lines redundant (always true), so this patch removes them
entirely.
---
 clang/lib/Headers/bmiintrin.h       |   4 -
 clang/lib/Headers/immintrin.h       | 224 ------------------------------------
 clang/lib/Headers/keylockerintrin.h |   9 --
 clang/lib/Headers/x86gprintrin.h    |  14 ---
 clang/lib/Headers/x86intrin.h       |  18 ---
 5 files changed, 269 deletions(-)

diff --git a/clang/lib/Headers/bmiintrin.h b/clang/lib/Headers/bmiintrin.h
index 59c5ece3977f..8024da55379c 100644
--- a/clang/lib/Headers/bmiintrin.h
+++ b/clang/lib/Headers/bmiintrin.h
@@ -161,8 +161,6 @@ _mm_tzcnt_64(unsigned long long __X) {
 
 #undef __RELAXED_FN_ATTRS
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__BMI__)
-
 /* Define the default attributes for the functions in this file. */
 #if defined(__cplusplus) && (__cplusplus >= 201103L)
 #define __DEFAULT_FN_ATTRS                                                     \
@@ -603,6 +601,4 @@ __blsr_u64(unsigned long long __X) {
 
 #undef __DEFAULT_FN_ATTRS
 
-#endif /* !defined(__SCE__) || __has_feature(modules) || defined(__BMI__) */
-
 #endif /* __BMIINTRIN_H */
diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h
index 19c5987257a2..35f012cc7004 100644
--- a/clang/lib/Headers/immintrin.h
+++ b/clang/lib/Headers/immintrin.h
@@ -16,231 +16,112 @@
 
 #include <x86gprintrin.h>
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__MMX__)
 #include <mmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SSE__)
 #include <xmmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SSE2__)
 #include <emmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SSE3__)
 #include <pmmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SSSE3__)
 #include <tmmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__SSE4_2__) || defined(__SSE4_1__))
 #include <smmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AES__) || defined(__PCLMUL__))
 #include <wmmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CLFLUSHOPT__)
 #include <clflushoptintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CLWB__)
 #include <clwbintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX__)
 #include <avxintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX2__)
 #include <avx2intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__F16C__)
 #include <f16cintrin.h>
-#endif
 
-/* No feature check desired due to internal checks */
 #include <bmiintrin.h>
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__BMI2__)
 #include <bmi2intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__LZCNT__)
 #include <lzcntintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__POPCNT__)
 #include <popcntintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__FMA__)
 #include <fmaintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512F__)
 #include <avx512fintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512VL__)
 #include <avx512vlintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512BW__)
 #include <avx512bwintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512BITALG__)
 #include <avx512bitalgintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512CD__)
 #include <avx512cdintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512VPOPCNTDQ__)
 #include <avx512vpopcntdqintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
 #include <avx512vpopcntdqvlintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512VNNI__)
 #include <avx512vnniintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512VNNI__))
 #include <avx512vlvnniintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVXVNNI__)
 #include <avxvnniintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512DQ__)
 #include <avx512dqintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512BITALG__))
 #include <avx512vlbitalgintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512BW__))
 #include <avx512vlbwintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512CD__))
 #include <avx512vlcdintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512DQ__))
 #include <avx512vldqintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512IFMA__)
 #include <avx512ifmaintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512IFMA__) && defined(__AVX512VL__))
 #include <avx512ifmavlintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVXIFMA__)
 #include <avxifmaintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512VBMI__)
 #include <avx512vbmiintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VBMI__) && defined(__AVX512VL__))
 #include <avx512vbmivlintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512VBMI2__)
 #include <avx512vbmi2intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
 #include <avx512vlvbmi2intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512FP16__)
 #include <avx512fp16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512FP16__))
 #include <avx512vlfp16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX512BF16__)
 #include <avx512bf16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512BF16__))
 #include <avx512vlbf16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__PKU__)
 #include <pkuintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__VPCLMULQDQ__)
 #include <vpclmulqdqintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__VAES__)
 #include <vaesintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__GFNI__)
 #include <gfniintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVXVNNIINT8__)
 #include <avxvnniint8intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVXNECONVERT__)
 #include <avxneconvertintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SHA512__)
 #include <sha512intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SM3__)
 #include <sm3intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SM4__)
 #include <sm4intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVXVNNIINT16__)
 #include <avxvnniint16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RDPID__)
 /// Reads the value of the IA32_TSC_AUX MSR (0xc0000103).
 ///
 /// \headerfile <immintrin.h>
@@ -252,9 +133,7 @@ static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __
 _rdpid_u32(void) {
   return __builtin_ia32_rdpid();
 }
-#endif // __RDPID__
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RDRND__)
 /// Returns a 16-bit hardware-generated random value.
 ///
 /// \headerfile <immintrin.h>
@@ -314,9 +193,7 @@ _rdrand64_step(unsigned long long *__p)
   }
 #endif
 }
-#endif /* __RDRND__ */
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__FSGSBASE__)
 #ifdef __x86_64__
 /// Reads the FS base register.
 ///
@@ -427,9 +304,6 @@ _writegsbase_u64(unsigned long long __V)
 }
 
 #endif
-#endif /* __FSGSBASE__ */
-
-#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVBE__)
 
 /* The structs used below are to force the load/store to be unaligned. This
  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
@@ -543,172 +417,86 @@ _storebe_i64(void * __P, long long __D) {
   ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D);
 }
 #endif
-#endif /* __MOVBE */
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RTM__)
 #include <rtmintrin.h>
 #include <xtestintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SHA__)
 #include <shaintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__FXSR__)
 #include <fxsrintrin.h>
-#endif
 
 /* No feature check desired due to internal MSC_VER checks */
 #include <xsaveintrin.h>
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__XSAVEOPT__)
 #include <xsaveoptintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__XSAVEC__)
 #include <xsavecintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__XSAVES__)
 #include <xsavesintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SHSTK__)
 #include <cetintrin.h>
-#endif
 
 /* Intrinsics inside adcintrin.h are available at all times. */
 #include <adcintrin.h>
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__ADX__)
 #include <adxintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RDSEED__)
 #include <rdseedintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__WBNOINVD__)
 #include <wbnoinvdintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CLDEMOTE__)
 #include <cldemoteintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__WAITPKG__)
 #include <waitpkgintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVDIRI__) ||     \
-    defined(__MOVDIR64B__)
 #include <movdirintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__MOVRS__)
 #include <movrsintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX10_2__) && defined(__MOVRS__))
 #include <movrs_avx10_2intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX10_2_512__) && defined(__MOVRS__))
 #include <movrs_avx10_2_512intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__PCONFIG__)
 #include <pconfigintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SGX__)
 #include <sgxintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__PTWRITE__)
 #include <ptwriteintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__INVPCID__)
 #include <invpcidintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__KL__) ||          \
-    defined(__WIDEKL__)
 #include <keylockerintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TILE__) ||    \
-    defined(__AMX_INT8__) || defined(__AMX_BF16__)
 #include <amxintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP16__)
 #include <amxfp16intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_COMPLEX__)
 #include <amxcomplexintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_FP8__)
 #include <amxfp8intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TRANSPOSE__)
 #include <amxtransposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_MOVRS__)
 #include <amxmovrsintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AMX_MOVRS__) && defined(__AMX_TRANSPOSE__))
 #include <amxmovrstransposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_AVX512__)
 #include <amxavx512intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_TF32__)
 #include <amxtf32intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AMX_TF32__) && defined(__AMX_TRANSPOSE__))
 #include <amxtf32transposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AMX_BF16__) && defined(__AMX_TRANSPOSE__))
 #include <amxbf16transposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AMX_FP16__) && defined(__AMX_TRANSPOSE__))
 #include <amxfp16transposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AMX_COMPLEX__) && defined(__AMX_TRANSPOSE__))
 #include <amxcomplextransposeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    defined(__AVX512VP2INTERSECT__)
 #include <avx512vp2intersectintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
 #include <avx512vlvp2intersectintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2__)
 #include <avx10_2bf16intrin.h>
 #include <avx10_2convertintrin.h>
 #include <avx10_2copyintrin.h>
@@ -716,33 +504,21 @@ _storebe_i64(void * __P, long long __D) {
 #include <avx10_2niintrin.h>
 #include <avx10_2satcvtdsintrin.h>
 #include <avx10_2satcvtintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__AVX10_2_512__)
 #include <avx10_2_512bf16intrin.h>
 #include <avx10_2_512convertintrin.h>
 #include <avx10_2_512minmaxintrin.h>
 #include <avx10_2_512niintrin.h>
 #include <avx10_2_512satcvtdsintrin.h>
 #include <avx10_2_512satcvtintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) ||                             \
-    (defined(__AVX10_2_512__) && defined(__SM4__))
 #include <sm4evexintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__ENQCMD__)
 #include <enqcmdintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SERIALIZE__)
 #include <serializeintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__TSXLDTRK__)
 #include <tsxldtrkintrin.h>
-#endif
 
 #if defined(_MSC_VER) && __has_extension(gnu_asm)
 /* Define the default attributes for these intrinsics */
diff --git a/clang/lib/Headers/keylockerintrin.h b/clang/lib/Headers/keylockerintrin.h
index f76e91b4d4b3..4e9e6bec20c0 100644
--- a/clang/lib/Headers/keylockerintrin.h
+++ b/clang/lib/Headers/keylockerintrin.h
@@ -28,8 +28,6 @@
 #ifndef _KEYLOCKERINTRIN_H
 #define _KEYLOCKERINTRIN_H
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__KL__)
-
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS \
   __attribute__((__always_inline__, __nodebug__, __target__("kl"),\
@@ -326,10 +324,6 @@ _mm_aesdec256kl_u8(__m128i* __odata, __m128i __idata, const void *__h) {
 
 #undef __DEFAULT_FN_ATTRS
 
-#endif /* !defined(__SCE__ || __has_feature(modules) || defined(__KL__) */
-
-#if !defined(__SCE__) || __has_feature(modules) || defined(__WIDEKL__)
-
 /* Define the default attributes for the functions in this file. */
 #define __DEFAULT_FN_ATTRS \
   __attribute__((__always_inline__, __nodebug__, __target__("kl,widekl"),\
@@ -521,7 +515,4 @@ _mm_aesdecwide256kl_u8(__m128i __odata[8], const __m128i __idata[8], const void*
 
 #undef __DEFAULT_FN_ATTRS
 
-#endif /* !defined(__SCE__) || __has_feature(modules) || defined(__WIDEKL__)   \
-        */
-
 #endif /* _KEYLOCKERINTRIN_H */
diff --git a/clang/lib/Headers/x86gprintrin.h b/clang/lib/Headers/x86gprintrin.h
index 3d5cc606d7e6..8d513ceffb6d 100644
--- a/clang/lib/Headers/x86gprintrin.h
+++ b/clang/lib/Headers/x86gprintrin.h
@@ -10,33 +10,19 @@
 #ifndef __X86GPRINTRIN_H
 #define __X86GPRINTRIN_H
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__HRESET__)
 #include <hresetintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__UINTR__)
 #include <uintrintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__USERMSR__)
 #include <usermsrintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CRC32__)
 #include <crc32intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__PRFCHI__)
 #include <prfchiintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RAOINT__)
 #include <raointintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CMPCCXADD__)
 #include <cmpccxaddintrin.h>
-#endif
 
 #if defined(__i386__)
 #define __SAVE_GPRBX "mov {%%ebx, %%eax |eax, ebx};"
diff --git a/clang/lib/Headers/x86intrin.h b/clang/lib/Headers/x86intrin.h
index f42e9e580f88..aaa84365ce3e 100644
--- a/clang/lib/Headers/x86intrin.h
+++ b/clang/lib/Headers/x86intrin.h
@@ -14,40 +14,22 @@
 
 #include <immintrin.h>
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__PRFCHW__)
 #include <prfchwintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__SSE4A__)
 #include <ammintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__FMA4__)
 #include <fma4intrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__XOP__)
 #include <xopintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__TBM__)
 #include <tbmintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__LWP__)
 #include <lwpintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__MWAITX__)
 #include <mwaitxintrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__CLZERO__)
 #include <clzerointrin.h>
-#endif
 
-#if !defined(__SCE__) || __has_feature(modules) || defined(__RDPRU__)
 #include <rdpruintrin.h>
-#endif
 
 #endif /* __X86INTRIN_H */
-- 
cgit v1.2.3


From 58c4fa96cb111ea8d399296838f4cb6a294115ca Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Wed, 18 Jun 2025 10:21:37 +0100
Subject: Fix bazel build for #142771 (#144659)

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index cb0f9d8c7413..c750eb733b3b 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -1650,6 +1650,7 @@ td_library(
     srcs = [
         "include/mlir/Dialect/EmitC/IR/EmitC.td",
         "include/mlir/Dialect/EmitC/IR/EmitCAttributes.td",
+        "include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td",
         "include/mlir/Dialect/EmitC/IR/EmitCBase.td",
         "include/mlir/Dialect/EmitC/IR/EmitCTypes.td",
     ],
@@ -1665,6 +1666,17 @@ td_library(
     ],
 )
 
+gentbl_cc_library(
+    name = "EmitCInterfacesIncGen",
+    tbl_outs = {
+        "include/mlir/Dialect/EmitC/IR/EmitCInterfaces.h.inc": ["-gen-op-interface-decls"],
+        "include/mlir/Dialect/EmitC/IR/EmitCInterfaces.cpp.inc": ["-gen-op-interface-defs"],
+    },
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/EmitC/IR/EmitCInterfaces.td",
+    deps = [":EmitCTdFiles"],
+)
+
 gentbl_cc_library(
     name = "EmitCAttributesIncGen",
     tbl_outs = {
@@ -3679,6 +3691,7 @@ cc_library(
         ":BytecodeOpInterface",
         ":CastInterfaces",
         ":ControlFlowInterfaces",
+        ":EmitCInterfacesIncGen",
         ":EmitCAttributesIncGen",
         ":EmitCOpsIncGen",
         ":FunctionInterfaces",
-- 
cgit v1.2.3


From 6fcdde2a4eb9eaf34511ac3a35075be329fe1fae Mon Sep 17 00:00:00 2001
From: Lucas Duarte Prates <lucas.prates@arm.com>
Date: Wed, 18 Jun 2025 10:26:46 +0100
Subject: [runtimes] Allow use of external llvm-lit on standalone builds
 (#144347)

When creating a standalone build of the runtimes sub-project, the
current CMake implementation looks for a lit executable that might
potentially exist in the build tree and unconditionally overrides the
value of `LLVM_EXTERNAL_LIT`. Due to this, any value passed via
`-DLLVM_EXTERNAL_LIT` when configuring the CMake project is ignored.
This change adds the `ALLOW_EXTERNAL` argument to the
`get_llvm_lit_path` call in the runtimes' CMakeLists.txt, allowing any
value previously set to be considered.
---
 runtimes/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 878b2eee3861..e4dd4ebfc678 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -259,7 +259,7 @@ if(LLVM_INCLUDE_TESTS)
     # dir rather than ${LLVM_INSTALL_DIR}/bin/llvm-lit (which may not exist if
     # LLVM_BINARY_DIR points at an installed LLVM tree rather than a build tree).
     set(LLVM_LIT_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/bin)
-    get_llvm_lit_path(_base_dir _file_name)
+    get_llvm_lit_path(_base_dir _file_name ALLOW_EXTERNAL)
     set(LLVM_EXTERNAL_LIT "${_base_dir}/${_file_name}" CACHE STRING "Command used to spawn lit" FORCE)
     # Avoid warning about missing llvm-lit from runtimes CMake files. This is
     # fine since we call configure_file() to create llvm-lit at the end of this
-- 
cgit v1.2.3


From 757a0e6d3b6130a984960ee413a3c8a6f99c7cb5 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Wed, 18 Jun 2025 11:29:23 +0200
Subject: [SystemZ] Treat FAKE_USE instructions as instructions without a size
 (#144390)

This patch fixes an error in which `FAKE_USE` instructions would trigger
an assertion in SystemZLongBranch due to them having a size of 0 without
being excepted in the assertion that each instruction, other than a set
of known 0-size instruction types, should have a non-0 size.

`FAKE_USE` instructions are no-op instructions that are emitted into
LLVM by the `-fextend-variable-liveness` clang flag to help preserve the
liveness of source variables in optimized code, and therefore they
should be understood as being valid size 0 instructions.
---
 llvm/lib/Target/SystemZ/SystemZLongBranch.cpp |  2 +-
 llvm/test/CodeGen/SystemZ/fake-use-size.ll    | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/SystemZ/fake-use-size.ll

diff --git a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
index 54e1eb095494..21a233b2ffa1 100644
--- a/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZLongBranch.cpp
@@ -215,7 +215,7 @@ static unsigned getInstSizeInBytes(const MachineInstr &MI,
           // These do not have a size:
           MI.isDebugOrPseudoInstr() || MI.isPosition() || MI.isKill() ||
           MI.isImplicitDef() || MI.getOpcode() == TargetOpcode::MEMBARRIER ||
-          MI.getOpcode() == TargetOpcode::INIT_UNDEF ||
+          MI.getOpcode() == TargetOpcode::INIT_UNDEF || MI.isFakeUse() ||
           // These have a size that may be zero:
           MI.isInlineAsm() || MI.getOpcode() == SystemZ::STACKMAP ||
           MI.getOpcode() == SystemZ::PATCHPOINT ||
diff --git a/llvm/test/CodeGen/SystemZ/fake-use-size.ll b/llvm/test/CodeGen/SystemZ/fake-use-size.ll
new file mode 100644
index 000000000000..1690a046aad4
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/fake-use-size.ll
@@ -0,0 +1,14 @@
+; RUN: llc -O0 < %s -mtriple=s390x-linux-gnu 2>&1 | FileCheck %s
+
+;; Tests that we can handle FAKE_USE instructions, emitting a comment for them
+;; in the resulting assembly.
+
+; CHECK:      .type   idd,@function
+; CHECK:      # %bb.0:
+; CHECK-NEXT: # fake_use:
+
+define double @idd(double %d) {
+entry:
+  notail call void (...) @llvm.fake.use(double %d)
+  ret double %d
+}
-- 
cgit v1.2.3


From bb00fd087a3c3e02fb812e41218ad0a85d9f0fe1 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Wed, 18 Jun 2025 09:35:13 +0000
Subject: [gn build] Port 669627d0c77e

---
 .../gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
index a06b2f11b452..4f3ef5bf174b 100644
--- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/cppcoreguidelines/BUILD.gn
@@ -45,6 +45,7 @@ static_library("cppcoreguidelines") {
     "RvalueReferenceParamNotMovedCheck.cpp",
     "SlicingCheck.cpp",
     "SpecialMemberFunctionsCheck.cpp",
+    "UseEnumClassCheck.cpp",
     "VirtualClassDestructorCheck.cpp",
   ]
 }
-- 
cgit v1.2.3


From cd8248f3e856a37cc1addcb74475b4d37dc8aa42 Mon Sep 17 00:00:00 2001
From: Scott Constable <scott.d.constable@intel.com>
Date: Wed, 18 Jun 2025 02:52:55 -0700
Subject: Fixed a bug in `-fsanitize-kcfi-arity` (#142867)

Compiling with `fsanitize-kcfi-arity` can crash the compiler if a
function has more than 6 arguments, including floating-point arguments
passed in XMM registers. This patch fixes the feature by only counter
integer and stack arguments toward kCFI arity. For example, the compiler
crashed when it attempted to generate kCFI arity information for this
function:
https://github.com/torvalds/linux/blob/16b70698aa3ae7888826d0c84567c72241cf6713/drivers/gpu/drm/amd/display/dc/dml/dcn32/display_mode_vba_util_32.h#L680

As noted in a comment, floating-point registers are not relevant to
enforcing kCFI at this time.
---
 llvm/lib/Target/X86/X86AsmPrinter.cpp | 23 ++++++++++++++++++-----
 llvm/test/CodeGen/X86/kcfi-arity.ll   | 29 +++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index 24eda602effd..c7238839c26b 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -198,14 +198,27 @@ void X86AsmPrinter::emitKCFITypeId(const MachineFunction &MF) {
     // Determine the function's arity (i.e., the number of arguments) at the ABI
     // level by counting the number of parameters that are passed
     // as registers, such as pointers and 64-bit (or smaller) integers. The
-    // Linux x86-64 ABI allows up to 6 parameters to be passed in GPRs.
+    // Linux x86-64 ABI allows up to 6 integer parameters to be passed in GPRs.
     // Additional parameters or parameters larger than 64 bits may be passed on
-    // the stack, in which case the arity is denoted as 7.
+    // the stack, in which case the arity is denoted as 7. Floating-point
+    // arguments passed in XMM0-XMM7 are not counted toward arity because
+    // floating-point values are not relevant to enforcing kCFI at this time.
     const unsigned ArityToRegMap[8] = {X86::EAX, X86::ECX, X86::EDX, X86::EBX,
                                        X86::ESP, X86::EBP, X86::ESI, X86::EDI};
-    int Arity = MF.getInfo<X86MachineFunctionInfo>()->getArgumentStackSize() > 0
-                    ? 7
-                    : MF.getRegInfo().liveins().size();
+    int Arity;
+    if (MF.getInfo<X86MachineFunctionInfo>()->getArgumentStackSize() > 0) {
+      Arity = 7;
+    } else {
+      Arity = 0;
+      for (const auto &LI : MF.getRegInfo().liveins()) {
+        auto Reg = LI.first;
+        if (X86::GR8RegClass.contains(Reg) || X86::GR16RegClass.contains(Reg) ||
+            X86::GR32RegClass.contains(Reg) ||
+            X86::GR64RegClass.contains(Reg)) {
+          ++Arity;
+        }
+      }
+    }
     DestReg = ArityToRegMap[Arity];
   }
 
diff --git a/llvm/test/CodeGen/X86/kcfi-arity.ll b/llvm/test/CodeGen/X86/kcfi-arity.ll
index 68d90adaf2a1..009fa7d2dc0a 100644
--- a/llvm/test/CodeGen/X86/kcfi-arity.ll
+++ b/llvm/test/CodeGen/X86/kcfi-arity.ll
@@ -192,9 +192,33 @@ entry:
   ret void
 }
 
+;; Ensure that floating-point values are not counted toward the arity
+; ASM-LABEL: __cfi_f12:
+; ASM: movl $2253188362, %ebp
+define dso_local void @f12(i32 noundef %v1, i32 noundef %v2, float noundef %v3, double noundef %v4, float noundef %v5, i32 noundef %v6, i32 noundef %v7, i32 noundef %v8) #0 !kcfi_type !7 {
+entry:
+  %v1.addr = alloca i32, align 4
+  %v2.addr = alloca i32, align 4
+  %v3.addr = alloca float, align 4
+  %v4.addr = alloca double, align 4
+  %v5.addr = alloca float, align 4
+  %v6.addr = alloca i32, align 4
+  %v7.addr = alloca i32, align 4
+  %v8.addr = alloca i32, align 4
+  store i32 %v1, ptr %v1.addr, align 4
+  store i32 %v2, ptr %v2.addr, align 4
+  store float %v3, ptr %v3.addr, align 4
+  store double %v4, ptr %v4.addr, align 4
+  store float %v5, ptr %v5.addr, align 4
+  store i32 %v6, ptr %v6.addr, align 4
+  store i32 %v7, ptr %v7.addr, align 4
+  store i32 %v8, ptr %v8.addr, align 4
+  ret void
+}
+
 attributes #0 = { "target-features"="+retpoline-indirect-branches,+retpoline-indirect-calls" }
 
-!llvm.module.flags = !{!0, !7}
+!llvm.module.flags = !{!0, !8}
 !0 = !{i32 4, !"kcfi", i32 1}
 !1 = !{i32 12345678}
 !2 = !{i32 4196274163}
@@ -202,4 +226,5 @@ attributes #0 = { "target-features"="+retpoline-indirect-branches,+retpoline-ind
 !4 = !{i32 199571451}
 !5 = !{i32 1046421190}
 !6 = !{i32 1342488295}
-!7 = !{i32 4, !"kcfi-arity", i32 1}
+!7 = !{i32 2253188362}
+!8 = !{i32 4, !"kcfi-arity", i32 1}
-- 
cgit v1.2.3


From dac0820b277835b7506a9c0d1dc5e077597f6742 Mon Sep 17 00:00:00 2001
From: Bjorn Pettersson <bjorn.a.pettersson@ericsson.com>
Date: Wed, 18 Jun 2025 11:37:51 +0200
Subject: [Thumb2] Regenerate some test checks. NFC

---
 llvm/test/CodeGen/Thumb2/mve-shuffle.ll    |  15 +-
 llvm/test/CodeGen/Thumb2/mve-vld3.ll       | 431 ++++++++++-------------------
 llvm/test/CodeGen/Thumb2/schedm7-hazard.ll |   1 +
 3 files changed, 159 insertions(+), 288 deletions(-)

diff --git a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
index 82c8d50e518b..94d5490cead2 100644
--- a/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-shuffle.ll
@@ -235,7 +235,7 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
 ; CHECK-LV-NEXT:    vmov.f32 s7, s1
 ; CHECK-LV-NEXT:    vmov q0, q1
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: shuffle3_i16:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    vmov q1, q0
@@ -248,6 +248,7 @@ define arm_aapcs_vfpcc <8 x i16> @shuffle3_i16(<8 x i16> %src) {
 ; CHECK-LIS-NEXT:    vmov.f32 s3, s5
 ; CHECK-LIS-NEXT:    vins.f16 s1, s7
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %out = shufflevector <8 x i16> %src, <8 x i16> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
   ret <8 x i16> %out
@@ -1170,7 +1171,7 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3_f16(<8 x half> %src) {
 ; CHECK-LV-NEXT:    vmov.f32 s7, s1
 ; CHECK-LV-NEXT:    vmov q0, q1
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: shuffle3_f16:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    vmov q1, q0
@@ -1183,6 +1184,7 @@ define arm_aapcs_vfpcc <8 x half> @shuffle3_f16(<8 x half> %src) {
 ; CHECK-LIS-NEXT:    vmov.f32 s3, s5
 ; CHECK-LIS-NEXT:    vins.f16 s1, s7
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %out = shufflevector <8 x half> %src, <8 x half> undef, <8 x i32> <i32 4, i32 5, i32 7, i32 6, i32 3, i32 1, i32 2, i32 0>
   ret <8 x half> %out
@@ -1514,7 +1516,7 @@ define arm_aapcs_vfpcc <8 x double> @shuffle9_f64(<4 x double> %src1, <4 x doubl
 ; CHECK-LV-NEXT:    vmov q1, q5
 ; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: shuffle9_f64:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
@@ -1534,6 +1536,7 @@ define arm_aapcs_vfpcc <8 x double> @shuffle9_f64(<4 x double> %src1, <4 x doubl
 ; CHECK-LIS-NEXT:    vmov q1, q5
 ; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %out = shufflevector <4 x double> %src1, <4 x double> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x double> %out
@@ -1627,7 +1630,7 @@ define arm_aapcs_vfpcc <8 x i64> @shuffle9_i64(<4 x i64> %src1, <4 x i64> %src2)
 ; CHECK-LV-NEXT:    vmov q1, q5
 ; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: shuffle9_i64:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
@@ -1647,6 +1650,7 @@ define arm_aapcs_vfpcc <8 x i64> @shuffle9_i64(<4 x i64> %src1, <4 x i64> %src2)
 ; CHECK-LIS-NEXT:    vmov q1, q5
 ; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %out = shufflevector <4 x i64> %src1, <4 x i64> %src2, <8 x i32> <i32 0, i32 4, i32 1, i32 5, i32 2, i32 6, i32 3, i32 7>
   ret <8 x i64> %out
@@ -1886,6 +1890,3 @@ entry:
   ret double %res
 }
 
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-LIS: {{.*}}
-; CHECK-LV: {{.*}}
diff --git a/llvm/test/CodeGen/Thumb2/mve-vld3.ll b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
index b6c8056891f8..4dd9173e2d41 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vld3.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vld3.ll
@@ -56,7 +56,7 @@ define void @vld3_v4i32(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LV-NEXT:    vpop {d8, d9}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: vld3_v4i32:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9}
@@ -80,6 +80,7 @@ define void @vld3_v4i32(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LIS-NEXT:    vpop {d8, d9}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %l1 = load <12 x i32>, ptr %src, align 4
   %s1 = shufflevector <12 x i32> %l1, <12 x i32> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -132,7 +133,7 @@ define void @vld3_v8i32(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
 ; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: vld3_v8i32:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
@@ -173,6 +174,7 @@ define void @vld3_v8i32(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
 ; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %l1 = load <24 x i32>, ptr %src, align 4
   %s1 = shufflevector <24 x i32> %l1, <24 x i32> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -259,7 +261,7 @@ define void @vld3_v16i32(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vstrw.32 q3, [r1, #32]
 ; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: vld3_v16i32:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
@@ -334,6 +336,7 @@ define void @vld3_v16i32(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vstrw.32 q3, [r1, #32]
 ; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %l1 = load <48 x i32>, ptr %src, align 4
   %s1 = shufflevector <48 x i32> %l1, <48 x i32> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -961,7 +964,7 @@ define void @vld3_v2i64(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vmov q0[3], q0[1], r7, r2
 ; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LV-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
-
+;
 ; CHECK-LIS-LABEL: vld3_v2i64:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .save {r4, r5, r6, r7, r8, lr}
@@ -991,6 +994,7 @@ define void @vld3_v2i64(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vmov q0[3], q0[1], r7, r2
 ; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LIS-NEXT:    pop.w {r4, r5, r6, r7, r8, pc}
+
 entry:
   %l1 = load <6 x i64>, ptr %src, align 4
   %s1 = shufflevector <6 x i64> %l1, <6 x i64> undef, <2 x i32> <i32 0, i32 3>
@@ -1147,7 +1151,7 @@ define void @vld3_v2f32(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vadd.f32 q0, q0, q1
 ; CHECK-LV-NEXT:    vstmia r1, {s0, s1}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: vld3_v2f32:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
@@ -1161,6 +1165,7 @@ define void @vld3_v2f32(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vadd.f32 q0, q1, q0
 ; CHECK-LIS-NEXT:    vstmia r1, {s0, s1}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %l1 = load <6 x float>, ptr %src, align 4
   %s1 = shufflevector <6 x float> %l1, <6 x float> undef, <2 x i32> <i32 0, i32 3>
@@ -1173,53 +1178,30 @@ entry:
 }
 
 define void @vld3_v4f32(ptr %src, ptr %dst) {
-; CHECK-LV-LABEL: vld3_v4f32:
-; CHECK-LV:       @ %bb.0: @ %entry
-; CHECK-LV-NEXT:    .vsave {d8, d9}
-; CHECK-LV-NEXT:    vpush {d8, d9}
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #32]
-; CHECK-LV-NEXT:    vmov.f32 s10, s2
-; CHECK-LV-NEXT:    vmov.f32 s13, s0
-; CHECK-LV-NEXT:    vmov.f32 s14, s3
-; CHECK-LV-NEXT:    vmov.f32 s8, s4
-; CHECK-LV-NEXT:    vmov.f32 s9, s7
-; CHECK-LV-NEXT:    vmov.f32 s12, s5
-; CHECK-LV-NEXT:    vmov.f32 s15, s18
-; CHECK-LV-NEXT:    vmov.f32 s11, s17
-; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LV-NEXT:    vmov.f32 s0, s6
-; CHECK-LV-NEXT:    vmov.f32 s2, s16
-; CHECK-LV-NEXT:    vmov.f32 s3, s19
-; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LV-NEXT:    vpop {d8, d9}
-; CHECK-LV-NEXT:    bx lr
+; CHECK-LABEL: vld3_v4f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9}
+; CHECK-NEXT:    vpush {d8, d9}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #16]
+; CHECK-NEXT:    vldrw.u32 q1, [r0]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #32]
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s8, s4
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s15, s18
+; CHECK-NEXT:    vmov.f32 s11, s17
+; CHECK-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vmov.f32 s3, s19
+; CHECK-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-NEXT:    vstrw.32 q0, [r1]
+; CHECK-NEXT:    vpop {d8, d9}
+; CHECK-NEXT:    bx lr
 
-; CHECK-LIS-LABEL: vld3_v4f32:
-; CHECK-LIS:       @ %bb.0: @ %entry
-; CHECK-LIS-NEXT:    .vsave {d8, d9}
-; CHECK-LIS-NEXT:    vpush {d8, d9}
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #16]
-; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0]
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #32]
-; CHECK-LIS-NEXT:    vmov.f32 s10, s2
-; CHECK-LIS-NEXT:    vmov.f32 s13, s0
-; CHECK-LIS-NEXT:    vmov.f32 s14, s3
-; CHECK-LIS-NEXT:    vmov.f32 s8, s4
-; CHECK-LIS-NEXT:    vmov.f32 s9, s7
-; CHECK-LIS-NEXT:    vmov.f32 s12, s5
-; CHECK-LIS-NEXT:    vmov.f32 s15, s18
-; CHECK-LIS-NEXT:    vmov.f32 s11, s17
-; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LIS-NEXT:    vmov.f32 s0, s6
-; CHECK-LIS-NEXT:    vmov.f32 s2, s16
-; CHECK-LIS-NEXT:    vmov.f32 s3, s19
-; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
-; CHECK-LIS-NEXT:    vpop {d8, d9}
-; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <12 x float>, ptr %src, align 4
   %s1 = shufflevector <12 x float> %l1, <12 x float> undef, <4 x i32> <i32 0, i32 3, i32 6, i32 9>
@@ -1232,87 +1214,47 @@ entry:
 }
 
 define void @vld3_v8f32(ptr %src, ptr %dst) {
-; CHECK-LV-LABEL: vld3_v8f32:
-; CHECK-LV:       @ %bb.0: @ %entry
-; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-LV-NEXT:    vmov.f32 s10, s2
-; CHECK-LV-NEXT:    vmov.f32 s13, s0
-; CHECK-LV-NEXT:    vmov.f32 s14, s3
-; CHECK-LV-NEXT:    vmov.f32 s8, s4
-; CHECK-LV-NEXT:    vmov.f32 s9, s7
-; CHECK-LV-NEXT:    vmov.f32 s12, s5
-; CHECK-LV-NEXT:    vmov.f32 s15, s18
-; CHECK-LV-NEXT:    vmov.f32 s11, s17
-; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LV-NEXT:    vmov.f32 s0, s6
-; CHECK-LV-NEXT:    vmov.f32 s2, s16
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-LV-NEXT:    vmov.f32 s3, s19
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-LV-NEXT:    vmov.f32 s17, s4
-; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LV-NEXT:    vmov.f32 s18, s7
-; CHECK-LV-NEXT:    vmov.f32 s22, s6
-; CHECK-LV-NEXT:    vmov.f32 s16, s9
-; CHECK-LV-NEXT:    vmov.f32 s19, s14
-; CHECK-LV-NEXT:    vmov.f32 s20, s8
-; CHECK-LV-NEXT:    vmov.f32 s21, s11
-; CHECK-LV-NEXT:    vmov.f32 s23, s13
-; CHECK-LV-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-LV-NEXT:    vmov.f32 s4, s10
-; CHECK-LV-NEXT:    vmov.f32 s6, s12
-; CHECK-LV-NEXT:    vmov.f32 s7, s15
-; CHECK-LV-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
-; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-LV-NEXT:    bx lr
+; CHECK-LABEL: vld3_v8f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s8, s4
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s15, s18
+; CHECK-NEXT:    vmov.f32 s11, s17
+; CHECK-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s3, s19
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vmov.f32 s19, s14
+; CHECK-NEXT:    vmov.f32 s20, s8
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s23, s13
+; CHECK-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11}
+; CHECK-NEXT:    bx lr
 
-; CHECK-LIS-LABEL: vld3_v8f32:
-; CHECK-LIS:       @ %bb.0: @ %entry
-; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11}
-; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11}
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-LIS-NEXT:    vmov.f32 s10, s2
-; CHECK-LIS-NEXT:    vmov.f32 s13, s0
-; CHECK-LIS-NEXT:    vmov.f32 s14, s3
-; CHECK-LIS-NEXT:    vmov.f32 s8, s4
-; CHECK-LIS-NEXT:    vmov.f32 s9, s7
-; CHECK-LIS-NEXT:    vmov.f32 s12, s5
-; CHECK-LIS-NEXT:    vmov.f32 s15, s18
-; CHECK-LIS-NEXT:    vmov.f32 s11, s17
-; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LIS-NEXT:    vmov.f32 s0, s6
-; CHECK-LIS-NEXT:    vmov.f32 s2, s16
-; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-LIS-NEXT:    vmov.f32 s3, s19
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-LIS-NEXT:    vmov.f32 s17, s4
-; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LIS-NEXT:    vmov.f32 s18, s7
-; CHECK-LIS-NEXT:    vmov.f32 s22, s6
-; CHECK-LIS-NEXT:    vmov.f32 s16, s9
-; CHECK-LIS-NEXT:    vmov.f32 s19, s14
-; CHECK-LIS-NEXT:    vmov.f32 s20, s8
-; CHECK-LIS-NEXT:    vmov.f32 s21, s11
-; CHECK-LIS-NEXT:    vmov.f32 s23, s13
-; CHECK-LIS-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-LIS-NEXT:    vmov.f32 s4, s10
-; CHECK-LIS-NEXT:    vmov.f32 s6, s12
-; CHECK-LIS-NEXT:    vmov.f32 s7, s15
-; CHECK-LIS-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
-; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11}
-; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <24 x float>, ptr %src, align 4
   %s1 = shufflevector <24 x float> %l1, <24 x float> undef, <8 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21>
@@ -1325,155 +1267,81 @@ entry:
 }
 
 define void @vld3_v16f32(ptr %src, ptr %dst) {
-; CHECK-LV-LABEL: vld3_v16f32:
-; CHECK-LV:       @ %bb.0: @ %entry
-; CHECK-LV-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LV-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LV-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-LV-NEXT:    vldrw.u32 q6, [r0, #176]
-; CHECK-LV-NEXT:    vmov.f32 s10, s2
-; CHECK-LV-NEXT:    vmov.f32 s13, s0
-; CHECK-LV-NEXT:    vmov.f32 s14, s3
-; CHECK-LV-NEXT:    vmov.f32 s8, s4
-; CHECK-LV-NEXT:    vmov.f32 s9, s7
-; CHECK-LV-NEXT:    vmov.f32 s12, s5
-; CHECK-LV-NEXT:    vmov.f32 s15, s18
-; CHECK-LV-NEXT:    vmov.f32 s11, s17
-; CHECK-LV-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LV-NEXT:    vmov.f32 s0, s6
-; CHECK-LV-NEXT:    vmov.f32 s2, s16
-; CHECK-LV-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-LV-NEXT:    vmov.f32 s3, s19
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-LV-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-LV-NEXT:    vmov.f32 s17, s4
-; CHECK-LV-NEXT:    vmov.f32 s18, s7
-; CHECK-LV-NEXT:    vmov.f32 s22, s6
-; CHECK-LV-NEXT:    vmov.f32 s16, s9
-; CHECK-LV-NEXT:    vmov.f32 s19, s14
-; CHECK-LV-NEXT:    vmov.f32 s20, s8
-; CHECK-LV-NEXT:    vmov.f32 s21, s11
-; CHECK-LV-NEXT:    vmov.f32 s23, s13
-; CHECK-LV-NEXT:    vmov.f32 s4, s10
-; CHECK-LV-NEXT:    vldrw.u32 q2, [r0, #160]
-; CHECK-LV-NEXT:    vmov.f32 s6, s12
-; CHECK-LV-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-LV-NEXT:    vmov.f32 s7, s15
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #144]
-; CHECK-LV-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-LV-NEXT:    vmov.f32 s18, s10
-; CHECK-LV-NEXT:    vmov.f32 s21, s8
-; CHECK-LV-NEXT:    vmov.f32 s22, s11
-; CHECK-LV-NEXT:    vmov.f32 s16, s12
-; CHECK-LV-NEXT:    vmov.f32 s17, s15
-; CHECK-LV-NEXT:    vmov.f32 s20, s13
-; CHECK-LV-NEXT:    vmov.f32 s23, s26
-; CHECK-LV-NEXT:    vmov.f32 s19, s25
-; CHECK-LV-NEXT:    vadd.f32 q4, q4, q5
-; CHECK-LV-NEXT:    vmov.f32 s8, s14
-; CHECK-LV-NEXT:    vmov.f32 s10, s24
-; CHECK-LV-NEXT:    vldrw.u32 q3, [r0, #112]
-; CHECK-LV-NEXT:    vmov.f32 s11, s27
-; CHECK-LV-NEXT:    vldrw.u32 q5, [r0, #128]
-; CHECK-LV-NEXT:    vadd.f32 q2, q4, q2
-; CHECK-LV-NEXT:    vldrw.u32 q4, [r0, #96]
-; CHECK-LV-NEXT:    vmov.f32 s25, s12
-; CHECK-LV-NEXT:    vstrw.32 q2, [r1, #48]
-; CHECK-LV-NEXT:    vmov.f32 s26, s15
-; CHECK-LV-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LV-NEXT:    vmov.f32 s30, s14
-; CHECK-LV-NEXT:    vstrw.32 q1, [r1]
-; CHECK-LV-NEXT:    vmov.f32 s24, s17
-; CHECK-LV-NEXT:    vmov.f32 s27, s22
-; CHECK-LV-NEXT:    vmov.f32 s28, s16
-; CHECK-LV-NEXT:    vmov.f32 s29, s19
-; CHECK-LV-NEXT:    vmov.f32 s31, s21
-; CHECK-LV-NEXT:    vadd.f32 q6, q7, q6
-; CHECK-LV-NEXT:    vmov.f32 s12, s18
-; CHECK-LV-NEXT:    vmov.f32 s14, s20
-; CHECK-LV-NEXT:    vmov.f32 s15, s23
-; CHECK-LV-NEXT:    vadd.f32 q3, q6, q3
-; CHECK-LV-NEXT:    vstrw.32 q3, [r1, #32]
-; CHECK-LV-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LV-NEXT:    bx lr
+; CHECK-LABEL: vld3_v16f32:
+; CHECK:       @ %bb.0: @ %entry
+; CHECK-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    vldrw.u32 q0, [r0, #64]
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #48]
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #80]
+; CHECK-NEXT:    vldrw.u32 q6, [r0, #176]
+; CHECK-NEXT:    vmov.f32 s10, s2
+; CHECK-NEXT:    vmov.f32 s13, s0
+; CHECK-NEXT:    vmov.f32 s14, s3
+; CHECK-NEXT:    vmov.f32 s8, s4
+; CHECK-NEXT:    vmov.f32 s9, s7
+; CHECK-NEXT:    vmov.f32 s12, s5
+; CHECK-NEXT:    vmov.f32 s15, s18
+; CHECK-NEXT:    vmov.f32 s11, s17
+; CHECK-NEXT:    vadd.f32 q2, q2, q3
+; CHECK-NEXT:    vmov.f32 s0, s6
+; CHECK-NEXT:    vmov.f32 s2, s16
+; CHECK-NEXT:    vldrw.u32 q1, [r0, #16]
+; CHECK-NEXT:    vmov.f32 s3, s19
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #32]
+; CHECK-NEXT:    vadd.f32 q0, q2, q0
+; CHECK-NEXT:    vldrw.u32 q2, [r0]
+; CHECK-NEXT:    vmov.f32 s17, s4
+; CHECK-NEXT:    vmov.f32 s18, s7
+; CHECK-NEXT:    vmov.f32 s22, s6
+; CHECK-NEXT:    vmov.f32 s16, s9
+; CHECK-NEXT:    vmov.f32 s19, s14
+; CHECK-NEXT:    vmov.f32 s20, s8
+; CHECK-NEXT:    vmov.f32 s21, s11
+; CHECK-NEXT:    vmov.f32 s23, s13
+; CHECK-NEXT:    vmov.f32 s4, s10
+; CHECK-NEXT:    vldrw.u32 q2, [r0, #160]
+; CHECK-NEXT:    vmov.f32 s6, s12
+; CHECK-NEXT:    vadd.f32 q4, q5, q4
+; CHECK-NEXT:    vmov.f32 s7, s15
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #144]
+; CHECK-NEXT:    vadd.f32 q1, q4, q1
+; CHECK-NEXT:    vmov.f32 s18, s10
+; CHECK-NEXT:    vmov.f32 s21, s8
+; CHECK-NEXT:    vmov.f32 s22, s11
+; CHECK-NEXT:    vmov.f32 s16, s12
+; CHECK-NEXT:    vmov.f32 s17, s15
+; CHECK-NEXT:    vmov.f32 s20, s13
+; CHECK-NEXT:    vmov.f32 s23, s26
+; CHECK-NEXT:    vmov.f32 s19, s25
+; CHECK-NEXT:    vadd.f32 q4, q4, q5
+; CHECK-NEXT:    vmov.f32 s8, s14
+; CHECK-NEXT:    vmov.f32 s10, s24
+; CHECK-NEXT:    vldrw.u32 q3, [r0, #112]
+; CHECK-NEXT:    vmov.f32 s11, s27
+; CHECK-NEXT:    vldrw.u32 q5, [r0, #128]
+; CHECK-NEXT:    vadd.f32 q2, q4, q2
+; CHECK-NEXT:    vldrw.u32 q4, [r0, #96]
+; CHECK-NEXT:    vmov.f32 s25, s12
+; CHECK-NEXT:    vstrw.32 q2, [r1, #48]
+; CHECK-NEXT:    vmov.f32 s26, s15
+; CHECK-NEXT:    vstrw.32 q0, [r1, #16]
+; CHECK-NEXT:    vmov.f32 s30, s14
+; CHECK-NEXT:    vstrw.32 q1, [r1]
+; CHECK-NEXT:    vmov.f32 s24, s17
+; CHECK-NEXT:    vmov.f32 s27, s22
+; CHECK-NEXT:    vmov.f32 s28, s16
+; CHECK-NEXT:    vmov.f32 s29, s19
+; CHECK-NEXT:    vmov.f32 s31, s21
+; CHECK-NEXT:    vadd.f32 q6, q7, q6
+; CHECK-NEXT:    vmov.f32 s12, s18
+; CHECK-NEXT:    vmov.f32 s14, s20
+; CHECK-NEXT:    vmov.f32 s15, s23
+; CHECK-NEXT:    vadd.f32 q3, q6, q3
+; CHECK-NEXT:    vstrw.32 q3, [r1, #32]
+; CHECK-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
+; CHECK-NEXT:    bx lr
 
-; CHECK-LIS-LABEL: vld3_v16f32:
-; CHECK-LIS:       @ %bb.0: @ %entry
-; CHECK-LIS-NEXT:    .vsave {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LIS-NEXT:    vpush {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LIS-NEXT:    vldrw.u32 q0, [r0, #64]
-; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #48]
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #80]
-; CHECK-LIS-NEXT:    vldrw.u32 q6, [r0, #176]
-; CHECK-LIS-NEXT:    vmov.f32 s10, s2
-; CHECK-LIS-NEXT:    vmov.f32 s13, s0
-; CHECK-LIS-NEXT:    vmov.f32 s14, s3
-; CHECK-LIS-NEXT:    vmov.f32 s8, s4
-; CHECK-LIS-NEXT:    vmov.f32 s9, s7
-; CHECK-LIS-NEXT:    vmov.f32 s12, s5
-; CHECK-LIS-NEXT:    vmov.f32 s15, s18
-; CHECK-LIS-NEXT:    vmov.f32 s11, s17
-; CHECK-LIS-NEXT:    vadd.f32 q2, q2, q3
-; CHECK-LIS-NEXT:    vmov.f32 s0, s6
-; CHECK-LIS-NEXT:    vmov.f32 s2, s16
-; CHECK-LIS-NEXT:    vldrw.u32 q1, [r0, #16]
-; CHECK-LIS-NEXT:    vmov.f32 s3, s19
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #32]
-; CHECK-LIS-NEXT:    vadd.f32 q0, q2, q0
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0]
-; CHECK-LIS-NEXT:    vmov.f32 s17, s4
-; CHECK-LIS-NEXT:    vmov.f32 s18, s7
-; CHECK-LIS-NEXT:    vmov.f32 s22, s6
-; CHECK-LIS-NEXT:    vmov.f32 s16, s9
-; CHECK-LIS-NEXT:    vmov.f32 s19, s14
-; CHECK-LIS-NEXT:    vmov.f32 s20, s8
-; CHECK-LIS-NEXT:    vmov.f32 s21, s11
-; CHECK-LIS-NEXT:    vmov.f32 s23, s13
-; CHECK-LIS-NEXT:    vmov.f32 s4, s10
-; CHECK-LIS-NEXT:    vldrw.u32 q2, [r0, #160]
-; CHECK-LIS-NEXT:    vmov.f32 s6, s12
-; CHECK-LIS-NEXT:    vadd.f32 q4, q5, q4
-; CHECK-LIS-NEXT:    vmov.f32 s7, s15
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #144]
-; CHECK-LIS-NEXT:    vadd.f32 q1, q4, q1
-; CHECK-LIS-NEXT:    vmov.f32 s18, s10
-; CHECK-LIS-NEXT:    vmov.f32 s21, s8
-; CHECK-LIS-NEXT:    vmov.f32 s22, s11
-; CHECK-LIS-NEXT:    vmov.f32 s16, s12
-; CHECK-LIS-NEXT:    vmov.f32 s17, s15
-; CHECK-LIS-NEXT:    vmov.f32 s20, s13
-; CHECK-LIS-NEXT:    vmov.f32 s23, s26
-; CHECK-LIS-NEXT:    vmov.f32 s19, s25
-; CHECK-LIS-NEXT:    vadd.f32 q4, q4, q5
-; CHECK-LIS-NEXT:    vmov.f32 s8, s14
-; CHECK-LIS-NEXT:    vmov.f32 s10, s24
-; CHECK-LIS-NEXT:    vldrw.u32 q3, [r0, #112]
-; CHECK-LIS-NEXT:    vmov.f32 s11, s27
-; CHECK-LIS-NEXT:    vldrw.u32 q5, [r0, #128]
-; CHECK-LIS-NEXT:    vadd.f32 q2, q4, q2
-; CHECK-LIS-NEXT:    vldrw.u32 q4, [r0, #96]
-; CHECK-LIS-NEXT:    vmov.f32 s25, s12
-; CHECK-LIS-NEXT:    vstrw.32 q2, [r1, #48]
-; CHECK-LIS-NEXT:    vmov.f32 s26, s15
-; CHECK-LIS-NEXT:    vstrw.32 q0, [r1, #16]
-; CHECK-LIS-NEXT:    vmov.f32 s30, s14
-; CHECK-LIS-NEXT:    vstrw.32 q1, [r1]
-; CHECK-LIS-NEXT:    vmov.f32 s24, s17
-; CHECK-LIS-NEXT:    vmov.f32 s27, s22
-; CHECK-LIS-NEXT:    vmov.f32 s28, s16
-; CHECK-LIS-NEXT:    vmov.f32 s29, s19
-; CHECK-LIS-NEXT:    vmov.f32 s31, s21
-; CHECK-LIS-NEXT:    vadd.f32 q6, q7, q6
-; CHECK-LIS-NEXT:    vmov.f32 s12, s18
-; CHECK-LIS-NEXT:    vmov.f32 s14, s20
-; CHECK-LIS-NEXT:    vmov.f32 s15, s23
-; CHECK-LIS-NEXT:    vadd.f32 q3, q6, q3
-; CHECK-LIS-NEXT:    vstrw.32 q3, [r1, #32]
-; CHECK-LIS-NEXT:    vpop {d8, d9, d10, d11, d12, d13, d14, d15}
-; CHECK-LIS-NEXT:    bx lr
 entry:
   %l1 = load <48 x float>, ptr %src, align 4
   %s1 = shufflevector <48 x float> %l1, <48 x float> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -1692,7 +1560,7 @@ define void @vld3_v16f16(ptr %src, ptr %dst) {
 ; CHECK-LV-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LV-NEXT:    vpop {d8, d9}
 ; CHECK-LV-NEXT:    bx lr
-
+;
 ; CHECK-LIS-LABEL: vld3_v16f16:
 ; CHECK-LIS:       @ %bb.0: @ %entry
 ; CHECK-LIS-NEXT:    .vsave {d8, d9}
@@ -1773,6 +1641,7 @@ define void @vld3_v16f16(ptr %src, ptr %dst) {
 ; CHECK-LIS-NEXT:    vstrw.32 q0, [r1]
 ; CHECK-LIS-NEXT:    vpop {d8, d9}
 ; CHECK-LIS-NEXT:    bx lr
+
 entry:
   %l1 = load <48 x half>, ptr %src, align 4
   %s1 = shufflevector <48 x half> %l1, <48 x half> undef, <16 x i32> <i32 0, i32 3, i32 6, i32 9, i32 12, i32 15, i32 18, i32 21, i32 24, i32 27, i32 30, i32 33, i32 36, i32 39, i32 42, i32 45>
@@ -1841,4 +1710,4 @@ entry:
   %a = fadd <4 x double> %a1, %s3
   store <4 x double> %a, ptr %dst
   ret void
-}
\ No newline at end of file
+}
diff --git a/llvm/test/CodeGen/Thumb2/schedm7-hazard.ll b/llvm/test/CodeGen/Thumb2/schedm7-hazard.ll
index d3e31d192d57..1b222b2a131a 100644
--- a/llvm/test/CodeGen/Thumb2/schedm7-hazard.ll
+++ b/llvm/test/CodeGen/Thumb2/schedm7-hazard.ll
@@ -17,6 +17,7 @@ define i32 @test(ptr %x0, i32 %y, i32 %z) {
 ; CHECK-NEXT:    adds r1, #1
 ; CHECK-NEXT:    muls r0, r1, r0
 ; CHECK-NEXT:    bx lr
+;
 ; NOBANK-LABEL: test:
 ; NOBANK:       @ %bb.0: @ %entry
 ; NOBANK-NEXT:    ldr r3, [r0]
-- 
cgit v1.2.3


From 5a9cc93a2058e2c26d766f7be6aee63e928bf825 Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Wed, 18 Jun 2025 10:57:03 +0100
Subject: Fix for bazel build #142079 (#144665)

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index c750eb733b3b..0b4441c15794 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -323,7 +323,6 @@ cc_library(
     ]) + [
         "include/mlir/IR/OpAsmOpInterface.h.inc",
         "include/mlir/IR/PDLPatternMatch.h.inc",
-        "include/mlir/Interfaces/CallInterfaces.h",
         "include/mlir/Interfaces/DataLayoutInterfaces.h",
         "include/mlir/Interfaces/InferIntRangeInterface.h",
         "include/mlir/Interfaces/SideEffectInterfaces.h",
@@ -332,6 +331,7 @@ cc_library(
         "include/mlir/IR/*.h",
     ]) + [
         "include/mlir/Interfaces/FoldInterfaces.h",
+        "include/mlir/Interfaces/CallInterfaces.h",
     ],
     includes = ["include"],
     deps = [
-- 
cgit v1.2.3


From a13b7cc00c5f4b9d2636ed7a22c1390cf8033baf Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 18 Jun 2025 12:24:55 +0200
Subject: [LICM] Support hoisting of non-argmemonly readonly calls (#144497)

The code checking whether a readonly call is safe to hoist is
currently limited to only argmemonly calls. However, the actual
implementation does not depend on this in any way. It either
does an MSSA clobber walk on the memory access (which will take
all locations accessed by the call into account), or it will
look at all MemoryDefs in an entirely location-independent manner.

The current restriction dates back to the time when LICM still
supported AST, in which case this code *did* reason about the
individual pointer arguments.
---
 llvm/lib/Transforms/Scalar/LICM.cpp        | 30 +++++-----------------------
 llvm/test/Transforms/LICM/call-hoisting.ll | 32 ++++++++++++++++++++++++++++++
 llvm/test/Transforms/LICM/funclet.ll       |  2 +-
 3 files changed, 38 insertions(+), 26 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index abb6ff1dcfe6..a6bb8b8a21b0 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -1117,13 +1117,6 @@ bool isHoistableAndSinkableInst(Instruction &I) {
           isa<ShuffleVectorInst>(I) || isa<ExtractValueInst>(I) ||
           isa<InsertValueInst>(I) || isa<FreezeInst>(I));
 }
-/// Return true if MSSA knows there are no MemoryDefs in the loop.
-bool isReadOnly(const MemorySSAUpdater &MSSAU, const Loop *L) {
-  for (auto *BB : L->getBlocks())
-    if (MSSAU.getMemorySSA()->getBlockDefs(BB))
-      return false;
-  return true;
-}
 
 /// Return true if I is the only Instruction with a MemoryAccess in L.
 bool isOnlyMemoryAccess(const Instruction *I, const Loop *L,
@@ -1234,24 +1227,11 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     if (Behavior.doesNotAccessMemory())
       return true;
     if (Behavior.onlyReadsMemory()) {
-      // A readonly argmemonly function only reads from memory pointed to by
-      // it's arguments with arbitrary offsets.  If we can prove there are no
-      // writes to this memory in the loop, we can hoist or sink.
-      if (Behavior.onlyAccessesArgPointees()) {
-        // TODO: expand to writeable arguments
-        for (Value *Op : CI->args())
-          if (Op->getType()->isPointerTy() &&
-              pointerInvalidatedByLoop(
-                  MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I,
-                  Flags, /*InvariantGroup=*/false))
-            return false;
-        return true;
-      }
-
-      // If this call only reads from memory and there are no writes to memory
-      // in the loop, we can hoist or sink the call as appropriate.
-      if (isReadOnly(MSSAU, CurLoop))
-        return true;
+      // If we can prove there are no writes to the memory read by the call, we
+      // can hoist or sink.
+      return !pointerInvalidatedByLoop(
+          MSSA, cast<MemoryUse>(MSSA->getMemoryAccess(CI)), CurLoop, I, Flags,
+          /*InvariantGroup=*/false);
     }
 
     // FIXME: This should use mod/ref information to see if we can hoist or
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
index 907f13438623..7124b4e445eb 100644
--- a/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -84,6 +84,38 @@ exit:
   ret void
 }
 
+declare i32 @load_not_argmemonly() readonly nounwind willreturn
+
+define void @test_load_not_argmemonly(ptr noalias %sink) {
+; CHECK-LABEL: define void @test_load_not_argmemonly(
+; CHECK-SAME: ptr noalias [[SINK:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @load_not_argmemonly()
+; CHECK-NEXT:    store i32 [[RET]], ptr [[SINK]], align 4
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  %ret = call i32 @load_not_argmemonly()
+  store i32 %ret, ptr %sink
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
 declare void @store(i32 %val, ptr %p) argmemonly writeonly nounwind
 
 define void @test(ptr %loc) {
diff --git a/llvm/test/Transforms/LICM/funclet.ll b/llvm/test/Transforms/LICM/funclet.ll
index 1cdd12ddc98e..03a49d8ddf69 100644
--- a/llvm/test/Transforms/LICM/funclet.ll
+++ b/llvm/test/Transforms/LICM/funclet.ll
@@ -153,6 +153,6 @@ else:                                             ; preds = %postinvoke
 
 declare void @may_throw()
 
-declare i32 @pure_computation() nounwind argmemonly readonly willreturn
+declare i32 @pure_computation() nounwind willreturn memory(none)
 
 declare i32 @__CxxFrameHandler3(...)
-- 
cgit v1.2.3


From ee4c2bb68752a6c4b463f3873cde278b8d348628 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Wed, 18 Jun 2025 11:32:22 +0100
Subject: [lldb][test] explicit-member-function-quals.cpp: add -glldb

This will get un-XFAILed but requires `-glldb` in an upcoming patch.
---
 lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp b/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp
index 5d1222795dd8..33001db69f83 100644
--- a/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp
+++ b/lldb/test/Shell/SymbolFile/DWARF/x86/explicit-member-function-quals.cpp
@@ -3,7 +3,7 @@
 // Tests that we correctly deduce the CV-quals and storage
 // class of explicit object member functions.
 //
-// RUN: %clangxx_host %s -target x86_64-pc-linux -g -std=c++23 -c -o %t
+// RUN: %clangxx_host %s -glldb -target x86_64-pc-linux -g -std=c++23 -c -o %t
 // RUN: %lldb %t -b -o "type lookup Foo" 2>&1 | FileCheck %s
 //
 // CHECK:      (lldb) type lookup Foo
-- 
cgit v1.2.3


From 561eca44e7639ee8805d0bf65a59b9898d782538 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Wed, 18 Jun 2025 12:32:33 +0200
Subject: [PowerPC] Split tests into asm and mir parts (NFC)

To allow both to be generated.
---
 .../test/CodeGen/PowerPC/aix32-cc-abi-vaarg-mir.ll | 221 +++++++++
 llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll    | 346 ++++----------
 .../test/CodeGen/PowerPC/aix64-cc-abi-vaarg-mir.ll | 192 ++++++++
 llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll    | 532 +++++++--------------
 4 files changed, 686 insertions(+), 605 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg-mir.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg-mir.ll

diff --git a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg-mir.ll b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg-mir.ll
new file mode 100644
index 000000000000..3eef8d5ff90f
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg-mir.ll
@@ -0,0 +1,221 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O2 -mtriple powerpc-ibm-aix-xcoff -mcpu=ppc -stop-after=machine-cp -verify-machineinstrs < %s | FileCheck %s
+
+define i32 @int_va_arg(i32 %a, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: int_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r11 = ADDI %fixed-stack.0, 0
+  ; CHECK-NEXT:   STW killed renamable $r4, 0, %fixed-stack.0 :: (store (s32) into %fixed-stack.0)
+  ; CHECK-NEXT:   STW killed renamable $r6, 8, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW killed renamable $r7, 12, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW killed renamable $r8, 16, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW killed renamable $r9, 20, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW killed renamable $r10, 24, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW renamable $r11, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
+  ; CHECK-NEXT:   STW killed renamable $r11, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $r4 = ADDI %fixed-stack.0, 4
+  ; CHECK-NEXT:   STW renamable $r4, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
+  ; CHECK-NEXT:   renamable $r6 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur)
+  ; CHECK-NEXT:   STW killed renamable $r4, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $r4 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur2)
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r6, killed renamable $r3
+  ; CHECK-NEXT:   renamable $r4 = RLWINM killed renamable $r4, 1, 0, 30
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
+  ; CHECK-NEXT:   STW killed renamable $r5, 4, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 4)
+  ; CHECK-NEXT:   BLR implicit $lr, implicit $rm, implicit $r3
+entry:
+  %arg1 = alloca ptr, align 4
+  %arg2 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %argp.cur = load ptr, ptr %arg1, align 4
+  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+  store ptr %argp.next, ptr %arg1, align 4
+  %0 = load i32, ptr %argp.cur, align 4
+  %add = add nsw i32 %0, %a
+  %argp.cur2 = load ptr, ptr %arg2, align 4
+  %argp.next3 = getelementptr inbounds i8, ptr %argp.cur2, i32 4
+  store ptr %argp.next3, ptr %arg2, align 4
+  %1 = load i32, ptr %argp.cur2, align 4
+  %mul = shl i32 %1, 1
+  %add4 = add nsw i32 %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg1)
+  ret i32 %add4
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.va_start(ptr)
+declare void @llvm.va_copy(ptr, ptr)
+declare void @llvm.va_end(ptr)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+define i32 @int_stack_va_arg(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, ...) local_unnamed_addr {
+  ; CHECK-LABEL: name: int_stack_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r11 = LI 4
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r4, killed renamable $r3
+  ; CHECK-NEXT:   renamable $r4 = ADDI %fixed-stack.0, 0
+  ; CHECK-NEXT:   renamable $r4 = RLWIMI killed renamable $r4, killed renamable $r11, 0, 29, 29
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r5
+  ; CHECK-NEXT:   STW killed renamable $r4, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r6
+  ; CHECK-NEXT:   renamable $r4 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur9, align 8)
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r7
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r8
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r9
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r10
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r4
+  ; CHECK-NEXT:   renamable $r4 = RLWINM killed renamable $r4, 1, 0, 30
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
+  ; CHECK-NEXT:   BLR implicit $lr, implicit $rm, implicit $r3
+entry:
+  %arg1 = alloca ptr, align 4
+  %arg2 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = add nsw i32 %two, %one
+  %add2 = add nsw i32 %add, %three
+  %add3 = add nsw i32 %add2, %four
+  %add4 = add nsw i32 %add3, %five
+  %add5 = add nsw i32 %add4, %six
+  %add6 = add nsw i32 %add5, %seven
+  %add7 = add nsw i32 %add6, %eight
+  %argp.cur = load ptr, ptr %arg1, align 4
+  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4
+  store ptr %argp.next, ptr %arg1, align 4
+  %0 = load i32, ptr %argp.cur, align 4
+  %add8 = add nsw i32 %add7, %0
+  %argp.cur9 = load ptr, ptr %arg2, align 4
+  %argp.next10 = getelementptr inbounds i8, ptr %argp.cur9, i32 4
+  store ptr %argp.next10, ptr %arg2, align 4
+  %1 = load i32, ptr %argp.cur9, align 4
+  %mul = shl i32 %1, 1
+  %add11 = add nsw i32 %add8, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg1)
+  ret i32 %add11
+}
+
+define double @double_va_arg(double %a, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: double_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $f1, $r5, $r6, $r7, $r8, $r9, $r10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r3 = ADDI %fixed-stack.0, 0
+  ; CHECK-NEXT:   STW killed renamable $r7, 8, %fixed-stack.0 :: (store (s32), align 8)
+  ; CHECK-NEXT:   STW renamable $r5, 0, %fixed-stack.0 :: (store (s32) into %fixed-stack.0, align 16)
+  ; CHECK-NEXT:   STW renamable $r6, 4, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 4)
+  ; CHECK-NEXT:   STW killed renamable $r8, 12, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW killed renamable $r9, 16, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 16, align 16)
+  ; CHECK-NEXT:   STW killed renamable $r10, 20, %fixed-stack.0 :: (store (s32))
+  ; CHECK-NEXT:   STW renamable $r3, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
+  ; CHECK-NEXT:   STW killed renamable $r3, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
+  ; CHECK-NEXT:   STW renamable $r5, 0, %stack.2 :: (store (s32) into %stack.2, align 8)
+  ; CHECK-NEXT:   STW renamable $r6, 4, %stack.2 :: (store (s32) into %stack.2 + 4)
+  ; CHECK-NEXT:   renamable $f0 = LFD 0, %stack.2 :: (load (s64) from %stack.2)
+  ; CHECK-NEXT:   STW killed renamable $r5, 0, %stack.3 :: (store (s32) into %stack.3, align 8)
+  ; CHECK-NEXT:   STW killed renamable $r6, 4, %stack.3 :: (store (s32) into %stack.3 + 4)
+  ; CHECK-NEXT:   renamable $f2 = LFD 0, %stack.3 :: (load (s64) from %stack.3)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   BLR implicit $lr, implicit $rm, implicit $f1
+entry:
+  %arg1 = alloca ptr, align 4
+  %arg2 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %argp.cur = load ptr, ptr %arg1, align 4
+  %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 8
+  store ptr %argp.next, ptr %arg1, align 4
+  %0 = load double, ptr %argp.cur, align 4
+  %add = fadd double %0, %a
+  %argp.cur2 = load ptr, ptr %arg2, align 4
+  %argp.next3 = getelementptr inbounds i8, ptr %argp.cur2, i32 8
+  store ptr %argp.next3, ptr %arg2, align 4
+  %1 = load double, ptr %argp.cur2, align 4
+  %mul = fmul double %1, 2.000000e+00
+  %add4 = fadd double %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg1)
+  ret double %add4
+}
+
+define double @double_stack_va_arg(double %one, double %two, double %three, double %four, double %five, double %six, double %seven, double %eight, double %nine, double %ten, double %eleven, double %twelve, double %thirteen, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: double_stack_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r3 = ADDI %fixed-stack.0, 0
+  ; CHECK-NEXT:   STW killed renamable $r3, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
+  ; CHECK-NEXT:   renamable $r3 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur142, align 16)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f1, killed renamable $f2, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f3, implicit $rm
+  ; CHECK-NEXT:   STW renamable $r3, 0, %stack.2 :: (store (s32) into %stack.2, align 8)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f4, implicit $rm
+  ; CHECK-NEXT:   renamable $r4 = LWZ 4, %fixed-stack.0 :: (load (s32) from %ir.argp.cur142 + 4)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f5, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f6, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f7, implicit $rm
+  ; CHECK-NEXT:   STW renamable $r4, 4, %stack.2 :: (store (s32) into %stack.2 + 4)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f8, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = LFD 0, %stack.2 :: (load (s64) from %stack.2)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f9, implicit $rm
+  ; CHECK-NEXT:   STW killed renamable $r3, 0, %stack.3 :: (store (s32) into %stack.3, align 8)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f10, implicit $rm
+  ; CHECK-NEXT:   STW killed renamable $r4, 4, %stack.3 :: (store (s32) into %stack.3 + 4)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f11, implicit $rm
+  ; CHECK-NEXT:   renamable $f2 = LFD 0, %stack.3 :: (load (s64) from %stack.3)
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f12, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f13, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   BLR implicit $lr, implicit $rm, implicit $f1
+entry:
+  %arg1 = alloca ptr, align 4
+  %arg2 = alloca ptr, align 4
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = fadd double %one, %two
+  %add2 = fadd double %add, %three
+  %add3 = fadd double %add2, %four
+  %add4 = fadd double %add3, %five
+  %add5 = fadd double %add4, %six
+  %add6 = fadd double %add5, %seven
+  %add7 = fadd double %add6, %eight
+  %add8 = fadd double %add7, %nine
+  %add9 = fadd double %add8, %ten
+  %add10 = fadd double %add9, %eleven
+  %add11 = fadd double %add10, %twelve
+  %add12 = fadd double %add11, %thirteen
+  %argp.cur1 = load ptr, ptr %arg1, align 4
+  %0 = load double, ptr %argp.cur1, align 4
+  %add13 = fadd double %add12, %0
+  %argp.cur142 = load ptr, ptr %arg2, align 4
+  %1 = load double, ptr %argp.cur142, align 4
+  %mul = fmul double %1, 2.000000e+00
+  %add16 = fadd double %add13, %mul
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg1)
+  ret double %add16
+}
diff --git a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
index 3c1b28a4eff1..6ec56ffe3e25 100644
--- a/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/aix32-cc-abi-vaarg.ll
@@ -1,29 +1,28 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -O2 -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck --check-prefix=ASM32 %s
-; RUN: llc -O2 -mtriple powerpc-ibm-aix-xcoff -mcpu=ppc -stop-after=machine-cp -verify-machineinstrs < %s | FileCheck --check-prefix=32BIT %s
+; RUN: llc -O2 -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec -mtriple powerpc-ibm-aix-xcoff < %s | FileCheck %s
 
 define i32 @int_va_arg(i32 %a, ...) local_unnamed_addr  {
-; ASM32-LABEL: int_va_arg:
-; ASM32:       # %bb.0: # %entry
-; ASM32-NEXT:    addi 11, 1, 28
-; ASM32-NEXT:    stw 4, 28(1)
-; ASM32-NEXT:    addi 4, 1, 32
-; ASM32-NEXT:    stw 6, 36(1)
-; ASM32-NEXT:    stw 11, -4(1)
-; ASM32-NEXT:    stw 11, -8(1)
-; ASM32-NEXT:    stw 4, -4(1)
-; ASM32-NEXT:    lwz 6, 28(1)
-; ASM32-NEXT:    stw 4, -8(1)
-; ASM32-NEXT:    add 3, 6, 3
-; ASM32-NEXT:    lwz 4, 28(1)
-; ASM32-NEXT:    slwi 4, 4, 1
-; ASM32-NEXT:    stw 7, 40(1)
-; ASM32-NEXT:    add 3, 3, 4
-; ASM32-NEXT:    stw 8, 44(1)
-; ASM32-NEXT:    stw 9, 48(1)
-; ASM32-NEXT:    stw 10, 52(1)
-; ASM32-NEXT:    stw 5, 32(1)
-; ASM32-NEXT:    blr
+; CHECK-LABEL: int_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi 11, 1, 28
+; CHECK-NEXT:    stw 4, 28(1)
+; CHECK-NEXT:    addi 4, 1, 32
+; CHECK-NEXT:    stw 6, 36(1)
+; CHECK-NEXT:    stw 11, -4(1)
+; CHECK-NEXT:    stw 11, -8(1)
+; CHECK-NEXT:    stw 4, -4(1)
+; CHECK-NEXT:    lwz 6, 28(1)
+; CHECK-NEXT:    stw 4, -8(1)
+; CHECK-NEXT:    add 3, 6, 3
+; CHECK-NEXT:    lwz 4, 28(1)
+; CHECK-NEXT:    slwi 4, 4, 1
+; CHECK-NEXT:    stw 7, 40(1)
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    stw 8, 44(1)
+; CHECK-NEXT:    stw 9, 48(1)
+; CHECK-NEXT:    stw 10, 52(1)
+; CHECK-NEXT:    stw 5, 32(1)
+; CHECK-NEXT:    blr
 entry:
   %arg1 = alloca ptr, align 4
   %arg2 = alloca ptr, align 4
@@ -49,45 +48,6 @@ entry:
   ret i32 %add4
 }
 
-; 32BIT-LABEL:   name:            int_va_arg
-; 32BIT-LABEL:   liveins:
-; 32BIT-DAG:     - { reg: '$r3', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r4', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r5', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r6', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r7', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r8', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r9', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r10', virtual-reg: '' }
-
-; 32BIT-LABEL:   fixedStack:
-; 32BIT-DAG:     - { id: 0, type: default, offset: 28, size: 4
-
-; 32BIT-LABEL:   stack:
-; 32BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 4
-; 32BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 4
-
-; 32BIT-LABEL:   body:             |
-; 32BIT-DAG:     liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
-; 32BIT-DAG:     STW killed renamable $r4, 0, %fixed-stack.0 :: (store (s32) into %fixed-stack.0)
-; 32BIT-DAG:     STW killed renamable $r5, 4, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 4)
-; 32BIT-DAG:     STW killed renamable $r6, 8, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r7, 12, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r8, 16, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r9, 20, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r10, 24, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r4, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
-; 32BIT-DAG:     renamable $r4 = ADDI %fixed-stack.0, 4
-; 32BIT-DAG:     STW killed renamable $r11, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
-; 32BIT-DAG:     renamable $r11 = ADDI %fixed-stack.0, 0
-; 32BIT-DAG:     STW renamable $r11, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
-; 32BIT-DAG:     STW renamable $r4, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
-; 32BIT-DAG:     renamable $r6 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur)
-; 32BIT-DAG:     renamable $r4 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur2)
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r6, killed renamable $r3
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
-; 32BIT-DAG:     BLR implicit $lr, implicit $rm, implicit $r3
-
 declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
 declare void @llvm.va_start(ptr)
 declare void @llvm.va_copy(ptr, ptr)
@@ -95,24 +55,24 @@ declare void @llvm.va_end(ptr)
 declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
 
 define i32 @int_stack_va_arg(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, ...) local_unnamed_addr {
-; ASM32-LABEL: int_stack_va_arg:
-; ASM32:       # %bb.0: # %entry
-; ASM32-NEXT:    add 3, 4, 3
-; ASM32-NEXT:    lwz 4, 56(1)
-; ASM32-NEXT:    li 11, 4
-; ASM32-NEXT:    add 3, 3, 5
-; ASM32-NEXT:    addi 12, 1, 56
-; ASM32-NEXT:    add 3, 3, 6
-; ASM32-NEXT:    rlwimi 12, 11, 0, 29, 29
-; ASM32-NEXT:    stw 12, -4(1)
-; ASM32-NEXT:    add 3, 3, 7
-; ASM32-NEXT:    add 3, 3, 8
-; ASM32-NEXT:    add 3, 3, 9
-; ASM32-NEXT:    add 3, 3, 10
-; ASM32-NEXT:    add 3, 3, 4
-; ASM32-NEXT:    slwi 4, 4, 1
-; ASM32-NEXT:    add 3, 3, 4
-; ASM32-NEXT:    blr
+; CHECK-LABEL: int_stack_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    lwz 4, 56(1)
+; CHECK-NEXT:    li 11, 4
+; CHECK-NEXT:    add 3, 3, 5
+; CHECK-NEXT:    addi 12, 1, 56
+; CHECK-NEXT:    add 3, 3, 6
+; CHECK-NEXT:    rlwimi 12, 11, 0, 29, 29
+; CHECK-NEXT:    stw 12, -4(1)
+; CHECK-NEXT:    add 3, 3, 7
+; CHECK-NEXT:    add 3, 3, 8
+; CHECK-NEXT:    add 3, 3, 9
+; CHECK-NEXT:    add 3, 3, 10
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    slwi 4, 4, 1
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
 entry:
   %arg1 = alloca ptr, align 4
   %arg2 = alloca ptr, align 4
@@ -145,63 +105,28 @@ entry:
   ret i32 %add11
 }
 
-; 32BIT-LABEL:   name:            int_stack_va_arg
-; 32BIT-LABEL:   liveins:
-; 32BIT-DAG:     - { reg: '$r3', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r4', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r5', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r6', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r7', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r8', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r9', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r10', virtual-reg: '' }
-
-; 32BIT-LABEL:   fixedStack:
-; 32BIT-DAG:     - { id: 0, type: default, offset: 56, size: 4
-
-; 32BIT-LABEL:   stack:
-; 32BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 4
-; 32BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 4
-
-; 32BIT-LABEL:   body:             |
-; 32BIT-DAG:     liveins: $r3, $r4, $r5, $r6, $r7, $r8, $r9, $r10
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r5
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r6
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r7
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r8
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r9
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r10
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r4, killed renamable $r3
-; 32BIT-DAG:     renamable $r4 = ADDI %fixed-stack.0, 0
-; 32BIT-DAG:     STW killed renamable $r4, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
-; 32BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r4
-; 32BIT-DAG:     renamable $r4 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur9, align 8)
-; 32BIT-DAG:     renamable $r11 = LI 4
-; 32BIT-DAG:     BLR implicit $lr, implicit $rm, implicit $r3
-
 define double @double_va_arg(double %a, ...) local_unnamed_addr  {
-; ASM32-LABEL: double_va_arg:
-; ASM32:       # %bb.0: # %entry
-; ASM32-NEXT:    stw 5, -16(1)
-; ASM32-NEXT:    addi 3, 1, 32
-; ASM32-NEXT:    stw 6, -12(1)
-; ASM32-NEXT:    lfd 0, -16(1)
-; ASM32-NEXT:    stw 5, -24(1)
-; ASM32-NEXT:    fadd 0, 0, 1
-; ASM32-NEXT:    stw 6, -20(1)
-; ASM32-NEXT:    lfd 1, -24(1)
-; ASM32-NEXT:    fadd 1, 1, 1
-; ASM32-NEXT:    stw 7, 40(1)
-; ASM32-NEXT:    fadd 1, 0, 1
-; ASM32-NEXT:    stw 5, 32(1)
-; ASM32-NEXT:    stw 6, 36(1)
-; ASM32-NEXT:    stw 8, 44(1)
-; ASM32-NEXT:    stw 9, 48(1)
-; ASM32-NEXT:    stw 10, 52(1)
-; ASM32-NEXT:    stw 3, -4(1)
-; ASM32-NEXT:    stw 3, -8(1)
-; ASM32-NEXT:    blr
+; CHECK-LABEL: double_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    stw 5, -16(1)
+; CHECK-NEXT:    addi 3, 1, 32
+; CHECK-NEXT:    stw 6, -12(1)
+; CHECK-NEXT:    lfd 0, -16(1)
+; CHECK-NEXT:    stw 5, -24(1)
+; CHECK-NEXT:    fadd 0, 0, 1
+; CHECK-NEXT:    stw 6, -20(1)
+; CHECK-NEXT:    lfd 1, -24(1)
+; CHECK-NEXT:    fadd 1, 1, 1
+; CHECK-NEXT:    stw 7, 40(1)
+; CHECK-NEXT:    fadd 1, 0, 1
+; CHECK-NEXT:    stw 5, 32(1)
+; CHECK-NEXT:    stw 6, 36(1)
+; CHECK-NEXT:    stw 8, 44(1)
+; CHECK-NEXT:    stw 9, 48(1)
+; CHECK-NEXT:    stw 10, 52(1)
+; CHECK-NEXT:    stw 3, -4(1)
+; CHECK-NEXT:    stw 3, -8(1)
+; CHECK-NEXT:    blr
 entry:
   %arg1 = alloca ptr, align 4
   %arg2 = alloca ptr, align 4
@@ -227,74 +152,35 @@ entry:
   ret double %add4
 }
 
-; 32BIT-LABEL:   name:            double_va_arg
-; 32BIT-LABEL:   liveins:
-; 32BIT-DAG:     - { reg: '$f1', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r5', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r6', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r7', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r8', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r9', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$r10', virtual-reg: '' }
-
-; 32BIT-LABEL:   fixedStack:
-; 32BIT-DAG:     - { id: 0, type: default, offset: 32, size: 4
-
-; 32BIT-LABEL:   stack:
-; 32BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 4
-; 32BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 4
-
-; 32BIT-LABEL:   body:             |
-; 32BIT-DAG:     liveins: $f1, $r5, $r6, $r7, $r8, $r9, $r10
-; 32BIT-DAG:     renamable $r3 = ADDI %fixed-stack.0, 0
-; 32BIT-DAG:     STW killed renamable $r7, 8, %fixed-stack.0 :: (store (s32), align 8)
-; 32BIT-DAG:     STW renamable $r5, 0, %fixed-stack.0 :: (store (s32) into %fixed-stack.0, align 16)
-; 32BIT-DAG:     STW renamable $r6, 4, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 4)
-; 32BIT-DAG:     STW killed renamable $r8, 12, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW killed renamable $r9, 16, %fixed-stack.0 :: (store (s32) into %fixed-stack.0 + 16, align 16)
-; 32BIT-DAG:     STW killed renamable $r10, 20, %fixed-stack.0 :: (store (s32))
-; 32BIT-DAG:     STW renamable $r3, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
-; 32BIT-DAG:     STW killed renamable $r3, 0, %stack.1.arg2 :: (store (s32) into %ir.arg2)
-; 32BIT-DAG:     STW renamable $r5, 0, %stack.2 :: (store (s32) into %stack.2, align 8)
-; 32BIT-DAG:     STW renamable $r6, 4, %stack.2 :: (store (s32) into %stack.2 + 4)
-; 32BIT-DAG:     renamable $f0 = LFD 0, %stack.2 :: (load (s64) from %stack.2)
-; 32BIT-DAG:     STW killed renamable $r5, 0, %stack.3 :: (store (s32) into %stack.3, align 8)
-; 32BIT-DAG:     STW killed renamable $r6, 4, %stack.3 :: (store (s32) into %stack.3 + 4)
-; 32BIT-DAG:     renamable $f2 = LFD 0, %stack.3 :: (load (s64) from %stack.3)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 32BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
-; 32BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 32BIT-DAG:     BLR implicit $lr, implicit $rm, implicit $f1
-
 define double @double_stack_va_arg(double %one, double %two, double %three, double %four, double %five, double %six, double %seven, double %eight, double %nine, double %ten, double %eleven, double %twelve, double %thirteen, ...) local_unnamed_addr  {
-; ASM32-LABEL: double_stack_va_arg:
-; ASM32:       # %bb.0: # %entry
-; ASM32-NEXT:    fadd 0, 1, 2
-; ASM32-NEXT:    addi 3, 1, 128
-; ASM32-NEXT:    lwz 4, 132(1)
-; ASM32-NEXT:    fadd 0, 0, 3
-; ASM32-NEXT:    stw 3, -4(1)
-; ASM32-NEXT:    fadd 0, 0, 4
-; ASM32-NEXT:    lwz 3, 128(1)
-; ASM32-NEXT:    fadd 0, 0, 5
-; ASM32-NEXT:    stw 3, -16(1)
-; ASM32-NEXT:    fadd 0, 0, 6
-; ASM32-NEXT:    stw 4, -12(1)
-; ASM32-NEXT:    fadd 0, 0, 7
-; ASM32-NEXT:    lfd 1, -16(1)
-; ASM32-NEXT:    fadd 0, 0, 8
-; ASM32-NEXT:    stw 3, -24(1)
-; ASM32-NEXT:    fadd 0, 0, 9
-; ASM32-NEXT:    stw 4, -20(1)
-; ASM32-NEXT:    fadd 0, 0, 10
-; ASM32-NEXT:    fadd 0, 0, 11
-; ASM32-NEXT:    fadd 0, 0, 12
-; ASM32-NEXT:    fadd 0, 0, 13
-; ASM32-NEXT:    fadd 0, 0, 1
-; ASM32-NEXT:    lfd 1, -24(1)
-; ASM32-NEXT:    fadd 1, 1, 1
-; ASM32-NEXT:    fadd 1, 0, 1
-; ASM32-NEXT:    blr
+; CHECK-LABEL: double_stack_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fadd 0, 1, 2
+; CHECK-NEXT:    addi 3, 1, 128
+; CHECK-NEXT:    lwz 4, 132(1)
+; CHECK-NEXT:    fadd 0, 0, 3
+; CHECK-NEXT:    stw 3, -4(1)
+; CHECK-NEXT:    fadd 0, 0, 4
+; CHECK-NEXT:    lwz 3, 128(1)
+; CHECK-NEXT:    fadd 0, 0, 5
+; CHECK-NEXT:    stw 3, -16(1)
+; CHECK-NEXT:    fadd 0, 0, 6
+; CHECK-NEXT:    stw 4, -12(1)
+; CHECK-NEXT:    fadd 0, 0, 7
+; CHECK-NEXT:    lfd 1, -16(1)
+; CHECK-NEXT:    fadd 0, 0, 8
+; CHECK-NEXT:    stw 3, -24(1)
+; CHECK-NEXT:    fadd 0, 0, 9
+; CHECK-NEXT:    stw 4, -20(1)
+; CHECK-NEXT:    fadd 0, 0, 10
+; CHECK-NEXT:    fadd 0, 0, 11
+; CHECK-NEXT:    fadd 0, 0, 12
+; CHECK-NEXT:    fadd 0, 0, 13
+; CHECK-NEXT:    fadd 0, 0, 1
+; CHECK-NEXT:    lfd 1, -24(1)
+; CHECK-NEXT:    fadd 1, 1, 1
+; CHECK-NEXT:    fadd 1, 0, 1
+; CHECK-NEXT:    blr
 entry:
   %arg1 = alloca ptr, align 4
   %arg2 = alloca ptr, align 4
@@ -325,57 +211,3 @@ entry:
   call void @llvm.lifetime.end.p0(i64 4, ptr nonnull %arg1)
   ret double %add16
 }
-
-; 32BIT-LABEL:   name:            double_stack_va_arg
-; 32BIT-LABEL:   liveins:
-; 32BIT-DAG:     - { reg: '$f1', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f2', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f3', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f4', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f5', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f6', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f7', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f8', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f9', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f10', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f11', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f12', virtual-reg: '' }
-; 32BIT-DAG:     - { reg: '$f13', virtual-reg: '' }
-
-; 32BIT-LABEL:   fixedStack:
-; 32BIT-DAG:     - { id: 0, type: default, offset: 128, size: 4
-
-; 32BIT-LABEL:   stack:
-; 32BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 4, alignment: 4,
-; 32BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 4, alignment: 4,
-; 32BIT-DAG:     - { id: 2, name: '', type: default, offset: 0, size: 8, alignment: 8,
-; 32BIT-DAG:     - { id: 3, name: '', type: default, offset: 0, size: 8, alignment: 8,
-
-; 32BIT-LABEL:   body:             |
-; 32BIT-DAG:     liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13
-; 32BIT-DAG:     renamable $r3 = ADDI %fixed-stack.0, 0
-; 32BIT-DAG:     STW killed renamable $r3, 0, %stack.0.arg1 :: (store (s32) into %ir.arg1)
-; 32BIT-DAG:     renamable $r3 = LWZ 0, %fixed-stack.0 :: (load (s32) from %ir.argp.cur142, align 16)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f1, killed renamable $f2, implicit $rm
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f3, implicit $rm
-; 32BIT-DAG:     STW renamable $r3, 0, %stack.2 :: (store (s32) into %stack.2, align 8)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f4, implicit $rm
-; 32BIT-DAG:     renamable $r4 = LWZ 4, %fixed-stack.0 :: (load (s32) from %ir.argp.cur142 + 4)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f5, implicit $rm
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f6, implicit $rm
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f7, implicit $rm
-; 32BIT-DAG:     STW renamable $r4, 4, %stack.2 :: (store (s32) into %stack.2 + 4)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f8, implicit $rm
-; 32BIT-DAG:     renamable $f1 = LFD 0, %stack.2 :: (load (s64) from %stack.2)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f9, implicit $rm
-; 32BIT-DAG:     STW killed renamable $r3, 0, %stack.3 :: (store (s32) into %stack.3, align 8)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f10, implicit $rm
-; 32BIT-DAG:     STW killed renamable $r4, 4, %stack.3 :: (store (s32) into %stack.3 + 4)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f11, implicit $rm
-; 32BIT-DAG:     renamable $f2 = LFD 0, %stack.3 :: (load (s64) from %stack.3)
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f12, implicit $rm
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f13, implicit $rm
-; 32BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 32BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
-; 32BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 32BIT-DAG:     BLR implicit $lr, implicit $rm, implicit $f1
diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg-mir.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg-mir.ll
new file mode 100644
index 000000000000..4d7c6fb6fa31
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg-mir.ll
@@ -0,0 +1,192 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -O2 -mtriple powerpc64-ibm-aix-xcoff -mcpu=ppc -stop-after=machine-cp -verify-machineinstrs < %s | FileCheck %s
+
+define i32 @int_va_arg(i32 %a, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: int_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x11 = ADDI8 %fixed-stack.0, 0
+  ; CHECK-NEXT:   STD killed renamable $x6, 16, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x7, 24, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x8, 32, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x9, 40, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x10, 48, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD renamable $x11, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $x6 = LD 0, %stack.1.arg2 :: (load (s64) from %ir.arg2)
+  ; CHECK-NEXT:   renamable $x7 = disjoint ADDI8 %fixed-stack.0, 4
+  ; CHECK-NEXT:   renamable $r8 = LWZ 0, %fixed-stack.0 :: (load (s32) from %fixed-stack.0, align 8)
+  ; CHECK-NEXT:   renamable $x9 = ADDI8 renamable $x6, 4
+  ; CHECK-NEXT:   STD killed renamable $x4, 0, %fixed-stack.0 :: (store (s64) into %fixed-stack.0)
+  ; CHECK-NEXT:   STD killed renamable $x5, 8, %fixed-stack.0 :: (store (s64) into %fixed-stack.0 + 8)
+  ; CHECK-NEXT:   STD killed renamable $x11, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
+  ; CHECK-NEXT:   STD killed renamable $x7, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
+  ; CHECK-NEXT:   STD killed renamable $x9, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $r4 = LWZ 0, killed renamable $x6 :: (load (s32))
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r8, renamable $r3, implicit killed $x3
+  ; CHECK-NEXT:   renamable $r4 = RLWINM killed renamable $r4, 1, 0, 30
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4, implicit-def $x3
+  ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $x3
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %0 = va_arg ptr %arg1, i32
+  %add = add nsw i32 %0, %a
+  %1 = va_arg ptr %arg2, i32
+  %mul = shl i32 %1, 1
+  %add3 = add nsw i32 %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret i32 %add3
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.va_start(ptr)
+declare void @llvm.va_copy(ptr, ptr)
+declare void @llvm.va_end(ptr)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+define i32 @int_stack_va_arg(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, ...) local_unnamed_addr {
+  ; CHECK-LABEL: name: int_stack_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $r11 = LWZ 0, %fixed-stack.0 :: (load (s32) from %fixed-stack.0, align 16)
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 renamable $r4, renamable $r3, implicit killed $x3, implicit killed $x4
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r6, implicit killed $x6
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r7, implicit killed $x7
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r8, implicit killed $x8
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r9, implicit killed $x9
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r10, implicit killed $x10
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r11
+  ; CHECK-NEXT:   renamable $r4 = RLWINM killed renamable $r11, 1, 0, 30
+  ; CHECK-NEXT:   renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4, implicit-def $x3
+  ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $x3
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = add nsw i32 %two, %one
+  %add2 = add nsw i32 %add, %three
+  %add3 = add nsw i32 %add2, %four
+  %add4 = add nsw i32 %add3, %five
+  %add5 = add nsw i32 %add4, %six
+  %add6 = add nsw i32 %add5, %seven
+  %add7 = add nsw i32 %add6, %eight
+  %0 = va_arg ptr %arg1, i32
+  %add8 = add nsw i32 %add7, %0
+  %1 = va_arg ptr %arg2, i32
+  %mul = shl i32 %1, 1
+  %add10 = add nsw i32 %add8, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret i32 %add10
+}
+
+define double @double_va_arg(double %a, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: double_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $f1, $x4, $x5, $x6, $x7, $x8, $x9, $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $x3 = ADDI8 %fixed-stack.0, 0
+  ; CHECK-NEXT:   STD killed renamable $x6, 16, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x7, 24, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x8, 32, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x9, 40, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD killed renamable $x10, 48, %fixed-stack.0 :: (store (s64))
+  ; CHECK-NEXT:   STD renamable $x3, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $x6 = LD 0, %stack.1.arg2 :: (load (s64) from %ir.arg2)
+  ; CHECK-NEXT:   renamable $x7 = ADDI8 %fixed-stack.0, 8
+  ; CHECK-NEXT:   STD killed renamable $x4, 0, %fixed-stack.0 :: (store (s64) into %fixed-stack.0)
+  ; CHECK-NEXT:   STD killed renamable $x3, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
+  ; CHECK-NEXT:   STD killed renamable $x7, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
+  ; CHECK-NEXT:   renamable $f0 = LFD 0, %fixed-stack.0 :: (load (s64))
+  ; CHECK-NEXT:   renamable $x3 = ADDI8 renamable $x6, 8
+  ; CHECK-NEXT:   STD killed renamable $x5, 8, %fixed-stack.0 :: (store (s64) into %fixed-stack.0 + 8)
+  ; CHECK-NEXT:   STD killed renamable $x3, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
+  ; CHECK-NEXT:   renamable $f2 = LFD 0, killed renamable $x6 :: (load (s64))
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
+  ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $f1
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %0 = va_arg ptr %arg1, double
+  %add = fadd double %0, %a
+  %1 = va_arg ptr %arg2, double
+  %mul = fmul double %1, 2.000000e+00
+  %add3 = fadd double %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret double %add3
+}
+
+define double @double_stack_va_arg(double %one, double %two, double %three, double %four, double %five, double %six, double %seven, double %eight, double %nine, double %ten, double %eleven, double %twelve, double %thirteen, ...) local_unnamed_addr  {
+  ; CHECK-LABEL: name: double_stack_va_arg
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   renamable $f0 = LFD 0, %fixed-stack.0 :: (load (s64))
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f2, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f3, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f4, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f5, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f6, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f7, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f8, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f9, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f10, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f11, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f12, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f13, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, renamable $f0, implicit $rm
+  ; CHECK-NEXT:   renamable $f0 = nofpexcept FADD killed renamable $f0, renamable $f0, implicit $rm
+  ; CHECK-NEXT:   renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f0, implicit $rm
+  ; CHECK-NEXT:   BLR8 implicit $lr8, implicit $rm, implicit $f1
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = fadd double %one, %two
+  %add2 = fadd double %add, %three
+  %add3 = fadd double %add2, %four
+  %add4 = fadd double %add3, %five
+  %add5 = fadd double %add4, %six
+  %add6 = fadd double %add5, %seven
+  %add7 = fadd double %add6, %eight
+  %add8 = fadd double %add7, %nine
+  %add9 = fadd double %add8, %ten
+  %add10 = fadd double %add9, %eleven
+  %add11 = fadd double %add10, %twelve
+  %add12 = fadd double %add11, %thirteen
+  %0 = va_arg ptr %arg1, double
+  %add13 = fadd double %add12, %0
+  %1 = va_arg ptr %arg2, double
+  %mul = fmul double %1, 2.000000e+00
+  %add15 = fadd double %add13, %mul
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret double %add15
+}
diff --git a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
index 1b9c66ad0b23..87f46fe3aca8 100644
--- a/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
+++ b/llvm/test/CodeGen/PowerPC/aix64-cc-abi-vaarg.ll
@@ -1,349 +1,185 @@
-; RUN: llc -O2 -mtriple powerpc64-ibm-aix-xcoff -mcpu=ppc -stop-after=machine-cp -verify-machineinstrs < %s | \
-; RUN: FileCheck --check-prefix=64BIT %s
-
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
 ; RUN: llc -O2 -verify-machineinstrs -mcpu=pwr4 -mattr=-altivec \
-; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s | \
-; RUN: FileCheck --check-prefix=ASM64 %s
-
-  define i32 @int_va_arg(i32 %a, ...) local_unnamed_addr  {
-  entry:
-    %arg1 = alloca ptr, align 8
-    %arg2 = alloca ptr, align 8
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.va_start(ptr nonnull %arg1)
-    call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
-    %0 = va_arg ptr %arg1, i32
-    %add = add nsw i32 %0, %a
-    %1 = va_arg ptr %arg2, i32
-    %mul = shl i32 %1, 1
-    %add3 = add nsw i32 %add, %mul
-    call void @llvm.va_end(ptr nonnull %arg1)
-    call void @llvm.va_end(ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
-    ret i32 %add3
-  }
-
-  declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
-  declare void @llvm.va_start(ptr)
-  declare void @llvm.va_copy(ptr, ptr)
-  declare void @llvm.va_end(ptr)
-  declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
-
-; 64BIT-LABEL:   name:            int_va_arg
-; 64BIT-LABEL:   liveins:
-; 64BIT-DAG:     - { reg: '$x3', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x4', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x5', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x6', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x7', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x8', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x9', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x10', virtual-reg: '' }
-
-; 64BIT-LABEL:   fixedStack:
-; 64BIT-DAG:     - { id: 0, type: default, offset: 56, size: 8
-
-; 64BIT-LABEL:   stack:
-; 64BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 8
-; 64BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 8
-
-; 64BIT-LABEL:   body:             |
-; 64BIT-DAG:     bb.0.entry:
-; 64BIT-DAG:     liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
-; 64BIT-DAG:     STD killed renamable $x4, 0, %fixed-stack.0 :: (store (s64) into %fixed-stack.0)
-; 64BIT-DAG:     STD killed renamable $x5, 8, %fixed-stack.0 :: (store (s64) into %fixed-stack.0 + 8)
-; 64BIT-DAG:     STD killed renamable $x6, 16, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x7, 24, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x8, 32, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x9, 40, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x10, 48, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     renamable $x11 = ADDI8 %fixed-stack.0, 0
-; 64BIT-DAG:     STD renamable $x11, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
-; 64BIT-DAG:     renamable $x6 = LD 0, %stack.1.arg2 :: (load (s64) from %ir.arg2)
-; 64BIT-DAG:     renamable $x9 = ADDI8 renamable $x6, 4
-; 64BIT-DAG:     renamable $x7 = disjoint ADDI8 %fixed-stack.0, 4
-; 64BIT-DAG:     renamable $r8 = LWZ 0, %fixed-stack.0 :: (load (s32) from %fixed-stack.0, align 8)
-; 64BIT-DAG:     STD killed renamable $x11, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
-; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
-; 64BIT-DAG:     STD killed renamable $x9, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
-; 64BIT-DAG:     renamable $r4 = LWZ 0, killed renamable $x6 :: (load (s32))
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r8, renamable $r3, implicit killed $x3
-; 64BIT-DAG:     renamable $r4 = RLWINM killed renamable $r4, 1, 0, 30
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4, implicit-def $x3
-; 64BIT-DAG:     BLR8 implicit $lr8, implicit $rm, implicit $x3
-
-; ASM64-LABEL:   .int_va_arg:
-; ASM64-DAG:     std 4, 56(1)
-; ASM64-DAG:     addi 4, 1, 56
-; ASM64-DAG:     std 4, -16(1)
-; ASM64-DAG:     std 4, -8(1)
-; ASM64-DAG:     ld 4, -16(1)
-; ASM64-DAG:     std 5, 64(1)
-; ASM64-DAG:     addi 5, 1, 60
-; ASM64-DAG:     std 5, -8(1)
-; ASM64-DAG:     addi 5, 4, 4
-; ASM64-DAG:     std 6, 72(1)
-; ASM64-DAG:     std 7, 80(1)
-; ASM64-DAG:     std 8, 88(1)
-; ASM64-DAG:     std 9, 96(1)
-; ASM64-DAG:     std 10, 104(1)
-; ASM64-DAG:     std 5, -16(1)
-; ASM64-DAG:     lwz 11, 56(1)
-; ASM64-DAG:     lwz 4, 0(4)
-; ASM64-DAG:     add 3, 11, 3
-; ASM64-DAG:     slwi 4, 4, 1
-; ASM64-DAG:     add 3, 3, 4
-; ASM64-DAG:     blr
-
-  define i32 @int_stack_va_arg(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, ...) local_unnamed_addr {
-  entry:
-    %arg1 = alloca ptr, align 8
-    %arg2 = alloca ptr, align 8
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.va_start(ptr nonnull %arg1)
-    call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
-    %add = add nsw i32 %two, %one
-    %add2 = add nsw i32 %add, %three
-    %add3 = add nsw i32 %add2, %four
-    %add4 = add nsw i32 %add3, %five
-    %add5 = add nsw i32 %add4, %six
-    %add6 = add nsw i32 %add5, %seven
-    %add7 = add nsw i32 %add6, %eight
-    %0 = va_arg ptr %arg1, i32
-    %add8 = add nsw i32 %add7, %0
-    %1 = va_arg ptr %arg2, i32
-    %mul = shl i32 %1, 1
-    %add10 = add nsw i32 %add8, %mul
-    call void @llvm.va_end(ptr nonnull %arg1)
-    call void @llvm.va_end(ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
-    ret i32 %add10
-  }
-
-; 64BIT-LABEL:    name:            int_stack_va_arg
-; 64BIT-LABEL:    liveins:
-; 64BIT-DAG:       - { reg: '$x3', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x4', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x5', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x6', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x7', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x8', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x9', virtual-reg: '' }
-; 64BIT-DAG:       - { reg: '$x10', virtual-reg: '' }
-
-; 64BIT-LABEL:   fixedStack:
-; 64BIT-DAG:     - { id: 0, type: default, offset: 112, size: 8, alignment: 16, stack-id: default,
-
-; 64BIT-LABEL:   stack:
-; 64BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 8, alignment: 8,
-; 64BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 8, alignment: 8,
-
-; 64BIT-LABEL:   body:             |
-; 64BIT-DAG:     liveins: $x3, $x4, $x5, $x6, $x7, $x8, $x9, $x10
-; 64BIT-DAG:     renamable $r11 = LWZ 0, %fixed-stack.0 :: (load (s32) from %fixed-stack.0, align 16)
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 renamable $r4, renamable $r3, implicit killed $x3, implicit killed $x4
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r5, implicit killed $x5
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r6, implicit killed $x6
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r7, implicit killed $x7
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r8, implicit killed $x8
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r9, implicit killed $x9
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r10, implicit killed $x10
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, renamable $r11
-; 64BIT-DAG:     renamable $r3 = nsw ADD4 killed renamable $r3, killed renamable $r4, implicit-def $x3
-; 64BIT-DAG:     BLR8 implicit $lr8, implicit $rm, implicit $x3
-
-; ASM64-LABEL:   .int_stack_va_arg:
-; ASM64-DAG:     add 3, 4, 3
-; ASM64-DAG:     add 3, 3, 5
-; ASM64-DAG:     add 3, 3, 6
-; ASM64-DAG:     add 3, 3, 7
-; ASM64-DAG:     add 3, 3, 8
-; ASM64-DAG:     add 3, 3, 9
-; ASM64-DAG:     add 3, 3, 10
-; ASM64-DAG:     lwz 11, 112(1)
-; ASM64-DAG:     slwi 4, 11, 1
-; ASM64-DAG:     add 3, 3, 11
-; ASM64-DAG:     add 3, 3, 4
-; ASM64-DAG:     blr
-
-  define double @double_va_arg(double %a, ...) local_unnamed_addr  {
-  entry:
-    %arg1 = alloca ptr, align 8
-    %arg2 = alloca ptr, align 8
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.va_start(ptr nonnull %arg1)
-    call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
-    %0 = va_arg ptr %arg1, double
-    %add = fadd double %0, %a
-    %1 = va_arg ptr %arg2, double
-    %mul = fmul double %1, 2.000000e+00
-    %add3 = fadd double %add, %mul
-    call void @llvm.va_end(ptr nonnull %arg1)
-    call void @llvm.va_end(ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
-    ret double %add3
-  }
-
-; 64BIT-LABEL:   name:            double_va_arg
-; 64BIT-LABEL:   liveins:
-; 64BIT-DAG:     - { reg: '$f1', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x4', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x5', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x6', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x7', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x8', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x9', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$x10', virtual-reg: '' }
-
-; 64BIT-LABEL:   fixedStack:
-; 64BIT-DAG:     - { id: 0, type: default, offset: 56, size: 8
-
-; 64BIT-LABEL:   stack:
-; 64BIT-DAG:     - { id: 0, name: arg1, type: default, offset: 0, size: 8
-; 64BIT-DAG:     - { id: 1, name: arg2, type: default, offset: 0, size: 8
-
-; 64BIT-LABEL:   body:             |
-; 64BIT-DAG:     liveins: $f1, $x4, $x5, $x6, $x7, $x8, $x9, $x10
-; 64BIT-DAG:     renamable $x3 = ADDI8 %fixed-stack.0, 0
-; 64BIT-DAG:     STD killed renamable $x4, 0, %fixed-stack.0 :: (store (s64) into %fixed-stack.0)
-; 64BIT-DAG:     STD killed renamable $x5, 8, %fixed-stack.0 :: (store (s64) into %fixed-stack.0 + 8)
-; 64BIT-DAG:     STD killed renamable $x6, 16, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x7, 24, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x8, 32, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x9, 40, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD killed renamable $x10, 48, %fixed-stack.0 :: (store (s64))
-; 64BIT-DAG:     STD renamable $x3, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
-; 64BIT-DAG:     renamable $x6 = LD 0, %stack.1.arg2 :: (load (s64) from %ir.arg2)
-; 64BIT-DAG:     renamable $x7 = ADDI8 %fixed-stack.0, 8
-; 64BIT-DAG:     STD killed renamable $x3, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
-; 64BIT-DAG:     STD killed renamable $x7, 0, %stack.0.arg1 :: (store (s64) into %ir.arg1)
-; 64BIT-DAG:     renamable $f0 = LFD 0, %fixed-stack.0 :: (load (s64))
-; 64BIT-DAG:     renamable $x3 = ADDI8 renamable $x6, 8
-; 64BIT-DAG:     STD killed renamable $x3, 0, %stack.1.arg2 :: (store (s64) into %ir.arg2)
-; 64BIT-DAG:     renamable $f2 = LFD 0, killed renamable $x6 :: (load (s64))
-; 64BIT-DAG:     renamable $f0 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 64BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f2, renamable $f2, implicit $rm
-; 64BIT-DAG:     renamable $f1 = nofpexcept FADD killed renamable $f0, killed renamable $f1, implicit $rm
-; 64BIT-DAG:     BLR8 implicit $lr8, implicit $rm, implicit $f1
-
-; ASM64-LABEL:  .double_va_arg:
-; ASM64-DAG:    addi 3, 1, 56
-; ASM64-DAG:    std 4, 56(1)
-; ASM64-DAG:    std 3, -8(1)
-; ASM64-DAG:    std 3, -16(1)
-; ASM64-DAG:    addi 3, 1, 64
-; ASM64-DAG:    std 3, -8(1)
-; ASM64-DAG:    ld 3, -16(1)
-; ASM64-DAG:    lfd 0, 56(1)
-; ASM64-DAG:    addi 4, 3, 8
-; ASM64-DAG:    std 5, 64(1)
-; ASM64-DAG:    fadd 0, 0, 1
-; ASM64-DAG:    std 6, 72(1)
-; ASM64-DAG:    std 7, 80(1)
-; ASM64-DAG:    std 8, 88(1)
-; ASM64-DAG:    std 9, 96(1)
-; ASM64-DAG:    std 10, 104(1)
-; ASM64-DAG:    std 4, -16(1)
-; ASM64-DAG:    lfd 1, 0(3)
-; ASM64-DAG:    fadd 1, 1, 1
-; ASM64-DAG:    fadd 1, 0, 1
-; ASM64-DAG:    blr
-
-  define double @double_stack_va_arg(double %one, double %two, double %three, double %four, double %five, double %six, double %seven, double %eight, double %nine, double %ten, double %eleven, double %twelve, double %thirteen, ...) local_unnamed_addr  {
-  entry:
-    %arg1 = alloca ptr, align 8
-    %arg2 = alloca ptr, align 8
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
-    call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.va_start(ptr nonnull %arg1)
-    call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
-    %add = fadd double %one, %two
-    %add2 = fadd double %add, %three
-    %add3 = fadd double %add2, %four
-    %add4 = fadd double %add3, %five
-    %add5 = fadd double %add4, %six
-    %add6 = fadd double %add5, %seven
-    %add7 = fadd double %add6, %eight
-    %add8 = fadd double %add7, %nine
-    %add9 = fadd double %add8, %ten
-    %add10 = fadd double %add9, %eleven
-    %add11 = fadd double %add10, %twelve
-    %add12 = fadd double %add11, %thirteen
-    %0 = va_arg ptr %arg1, double
-    %add13 = fadd double %add12, %0
-    %1 = va_arg ptr %arg2, double
-    %mul = fmul double %1, 2.000000e+00
-    %add15 = fadd double %add13, %mul
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
-    call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
-    ret double %add15
-  }
-
-
-; 64BIT-LABEL:   name:            double_stack_va_arg
-; 64BIT-LABEL:   liveins:
-; 64BIT-DAG:     - { reg: '$f1', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f2', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f3', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f4', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f5', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f6', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f7', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f8', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f9', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f10', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f11', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f12', virtual-reg: '' }
-; 64BIT-DAG:     - { reg: '$f13', virtual-reg: '' }
-
-; 64BIT-LABEL:   fixedStack:
-; 64BIT-DAG:       - { id: 0, type: default, offset: 152, size: 8
-
-; 64BIT-LABEL:   stack:
-; 64BIT-DAG:       - { id: 0, name: arg1, type: default, offset: 0, size: 8
-; 64BIT-DAG:       - { id: 1, name: arg2, type: default, offset: 0, size: 8
-
-; 64BIT-LABEL:     body:             |
-; 64BIT-DAG:       liveins: $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13
-; 64BIT-DAG:       renamable $f0 = LFD 0, %fixed-stack.0 :: (load (s64))
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f2, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f3, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f4, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f5, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f6, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f7, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f8, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f9, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f10, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f11, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f12, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f13, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, renamable $f0, implicit $rm
-; 64BIT-DAG:       renamable $f0 = nofpexcept FADD killed renamable $f0, renamable $f0, implicit $rm
-; 64BIT-DAG:       renamable $f1 = nofpexcept FADD killed renamable $f1, killed renamable $f0, implicit $rm
-; 64BIT-DAG:       BLR8 implicit $lr8, implicit $rm, implicit $f1
-
-; ASM64-LABEL:   .double_stack_va_arg:
-; ASM64-DAG:     fadd 1, 1, 2
-; ASM64-DAG:     fadd 1, 1, 3
-; ASM64-DAG:     fadd 1, 1, 4
-; ASM64-DAG:     fadd 1, 1, 5
-; ASM64-DAG:     fadd 1, 1, 6
-; ASM64-DAG:     fadd 1, 1, 7
-; ASM64-DAG:     fadd 1, 1, 8
-; ASM64-DAG:     fadd 1, 1, 9
-; ASM64-DAG:     fadd 1, 1, 10
-; ASM64-DAG:     fadd 1, 1, 11
-; ASM64-DAG:     fadd 1, 1, 12
-; ASM64-DAG:     fadd 1, 1, 13
-; ASM64-DAG:     lfd 0, 152(1)
-; ASM64-DAG:     fadd 1, 1, 0
-; ASM64-DAG:     fadd 0, 0, 0
-; ASM64-DAG:     fadd 1, 1, 0
-; ASM64-DAG:     blr
+; RUN: -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s
+
+define i32 @int_va_arg(i32 %a, ...) local_unnamed_addr  {
+; CHECK-LABEL: int_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    std 4, 56(1)
+; CHECK-NEXT:    addi 4, 1, 56
+; CHECK-NEXT:    std 4, -16(1)
+; CHECK-NEXT:    std 4, -8(1)
+; CHECK-NEXT:    ld 4, -16(1)
+; CHECK-NEXT:    std 5, 64(1)
+; CHECK-NEXT:    addi 5, 1, 60
+; CHECK-NEXT:    std 5, -8(1)
+; CHECK-NEXT:    addi 5, 4, 4
+; CHECK-NEXT:    std 6, 72(1)
+; CHECK-NEXT:    std 7, 80(1)
+; CHECK-NEXT:    std 8, 88(1)
+; CHECK-NEXT:    std 9, 96(1)
+; CHECK-NEXT:    std 10, 104(1)
+; CHECK-NEXT:    std 5, -16(1)
+; CHECK-NEXT:    lwz 11, 56(1)
+; CHECK-NEXT:    lwz 4, 0(4)
+; CHECK-NEXT:    add 3, 11, 3
+; CHECK-NEXT:    slwi 4, 4, 1
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %0 = va_arg ptr %arg1, i32
+  %add = add nsw i32 %0, %a
+  %1 = va_arg ptr %arg2, i32
+  %mul = shl i32 %1, 1
+  %add3 = add nsw i32 %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret i32 %add3
+}
+
+declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.va_start(ptr)
+declare void @llvm.va_copy(ptr, ptr)
+declare void @llvm.va_end(ptr)
+declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+
+define i32 @int_stack_va_arg(i32 %one, i32 %two, i32 %three, i32 %four, i32 %five, i32 %six, i32 %seven, i32 %eight, ...) local_unnamed_addr {
+; CHECK-LABEL: int_stack_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    add 3, 4, 3
+; CHECK-NEXT:    lwz 11, 112(1)
+; CHECK-NEXT:    add 3, 3, 5
+; CHECK-NEXT:    add 3, 3, 6
+; CHECK-NEXT:    add 3, 3, 7
+; CHECK-NEXT:    add 3, 3, 8
+; CHECK-NEXT:    add 3, 3, 9
+; CHECK-NEXT:    add 3, 3, 10
+; CHECK-NEXT:    add 3, 3, 11
+; CHECK-NEXT:    slwi 4, 11, 1
+; CHECK-NEXT:    add 3, 3, 4
+; CHECK-NEXT:    blr
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = add nsw i32 %two, %one
+  %add2 = add nsw i32 %add, %three
+  %add3 = add nsw i32 %add2, %four
+  %add4 = add nsw i32 %add3, %five
+  %add5 = add nsw i32 %add4, %six
+  %add6 = add nsw i32 %add5, %seven
+  %add7 = add nsw i32 %add6, %eight
+  %0 = va_arg ptr %arg1, i32
+  %add8 = add nsw i32 %add7, %0
+  %1 = va_arg ptr %arg2, i32
+  %mul = shl i32 %1, 1
+  %add10 = add nsw i32 %add8, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret i32 %add10
+}
+
+define double @double_va_arg(double %a, ...) local_unnamed_addr  {
+; CHECK-LABEL: double_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    addi 3, 1, 56
+; CHECK-NEXT:    std 4, 56(1)
+; CHECK-NEXT:    std 3, -8(1)
+; CHECK-NEXT:    std 3, -16(1)
+; CHECK-NEXT:    addi 3, 1, 64
+; CHECK-NEXT:    std 3, -8(1)
+; CHECK-NEXT:    ld 3, -16(1)
+; CHECK-NEXT:    lfd 0, 56(1)
+; CHECK-NEXT:    addi 4, 3, 8
+; CHECK-NEXT:    std 5, 64(1)
+; CHECK-NEXT:    fadd 0, 0, 1
+; CHECK-NEXT:    std 6, 72(1)
+; CHECK-NEXT:    std 7, 80(1)
+; CHECK-NEXT:    std 8, 88(1)
+; CHECK-NEXT:    std 9, 96(1)
+; CHECK-NEXT:    std 10, 104(1)
+; CHECK-NEXT:    std 4, -16(1)
+; CHECK-NEXT:    lfd 1, 0(3)
+; CHECK-NEXT:    fadd 1, 1, 1
+; CHECK-NEXT:    fadd 1, 0, 1
+; CHECK-NEXT:    blr
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %0 = va_arg ptr %arg1, double
+  %add = fadd double %0, %a
+  %1 = va_arg ptr %arg2, double
+  %mul = fmul double %1, 2.000000e+00
+  %add3 = fadd double %add, %mul
+  call void @llvm.va_end(ptr nonnull %arg1)
+  call void @llvm.va_end(ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret double %add3
+}
+
+define double @double_stack_va_arg(double %one, double %two, double %three, double %four, double %five, double %six, double %seven, double %eight, double %nine, double %ten, double %eleven, double %twelve, double %thirteen, ...) local_unnamed_addr  {
+; CHECK-LABEL: double_stack_va_arg:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    fadd 1, 1, 2
+; CHECK-NEXT:    lfd 0, 152(1)
+; CHECK-NEXT:    fadd 1, 1, 3
+; CHECK-NEXT:    fadd 1, 1, 4
+; CHECK-NEXT:    fadd 1, 1, 5
+; CHECK-NEXT:    fadd 1, 1, 6
+; CHECK-NEXT:    fadd 1, 1, 7
+; CHECK-NEXT:    fadd 1, 1, 8
+; CHECK-NEXT:    fadd 1, 1, 9
+; CHECK-NEXT:    fadd 1, 1, 10
+; CHECK-NEXT:    fadd 1, 1, 11
+; CHECK-NEXT:    fadd 1, 1, 12
+; CHECK-NEXT:    fadd 1, 1, 13
+; CHECK-NEXT:    fadd 1, 1, 0
+; CHECK-NEXT:    fadd 0, 0, 0
+; CHECK-NEXT:    fadd 1, 1, 0
+; CHECK-NEXT:    blr
+entry:
+  %arg1 = alloca ptr, align 8
+  %arg2 = alloca ptr, align 8
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg1)
+  call void @llvm.lifetime.start.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.va_start(ptr nonnull %arg1)
+  call void @llvm.va_copy(ptr nonnull %arg2, ptr nonnull %arg1)
+  %add = fadd double %one, %two
+  %add2 = fadd double %add, %three
+  %add3 = fadd double %add2, %four
+  %add4 = fadd double %add3, %five
+  %add5 = fadd double %add4, %six
+  %add6 = fadd double %add5, %seven
+  %add7 = fadd double %add6, %eight
+  %add8 = fadd double %add7, %nine
+  %add9 = fadd double %add8, %ten
+  %add10 = fadd double %add9, %eleven
+  %add11 = fadd double %add10, %twelve
+  %add12 = fadd double %add11, %thirteen
+  %0 = va_arg ptr %arg1, double
+  %add13 = fadd double %add12, %0
+  %1 = va_arg ptr %arg2, double
+  %mul = fmul double %1, 2.000000e+00
+  %add15 = fadd double %add13, %mul
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg2)
+  call void @llvm.lifetime.end.p0(i64 8, ptr nonnull %arg1)
+  ret double %add15
+}
-- 
cgit v1.2.3


From acde20b5605f3a3a8da2217e4526fc045e6603ed Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Wed, 18 Jun 2025 06:39:52 -0400
Subject: [HLSL][SPIRV] Add vk::constant_id attribute. (#143544)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The vk::constant_id attribute is used to indicate that a global const
variable
represents a specialization constant in SPIR-V. This PR adds this
attribute to clang.

The documentation for the attribute is
[here](https://github.com/microsoft/DirectXShaderCompiler/blob/main/docs/SPIR-V.rst#specialization-constants).

The strategy is to to modify the initializer to get the value of a
specialize constant for a builtin defined in the SPIR-V backend.

Implements https://github.com/llvm/wg-hlsl/pull/287

Fixes https://github.com/llvm/llvm-project/issues/142448

---------

Co-authored-by: Nathan Gauër <github@keenuts.net>
---
 clang/include/clang/Basic/Attr.td                  |   8 +
 clang/include/clang/Basic/AttrDocs.td              |  15 ++
 clang/include/clang/Basic/Builtins.td              |  13 ++
 clang/include/clang/Basic/DiagnosticSemaKinds.td   |   4 +
 clang/include/clang/Sema/SemaHLSL.h                |   5 +-
 clang/lib/CodeGen/CGHLSLBuiltins.cpp               |  74 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h                |   6 +
 clang/lib/Sema/SemaDecl.cpp                        |  13 ++
 clang/lib/Sema/SemaDeclAttr.cpp                    |   3 +
 clang/lib/Sema/SemaHLSL.cpp                        | 120 +++++++++++-
 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl    | 130 +++++++++++++
 .../inline-spirv/SpirvType.alignment.hlsl          |  16 --
 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl |  68 -------
 .../vk-features/SpirvType.alignment.hlsl           |  16 ++
 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl  |  68 +++++++
 .../CodeGenHLSL/vk-features/vk.spec-constant.hlsl  | 210 +++++++++++++++++++++
 clang/test/SemaHLSL/vk.spec-constant.error.hlsl    |  37 ++++
 17 files changed, 720 insertions(+), 86 deletions(-)
 create mode 100644 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
 create mode 100644 clang/test/SemaHLSL/vk.spec-constant.error.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index f113cd2ba2fb..27fea7dea0a5 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -5023,6 +5023,14 @@ def HLSLVkExtBuiltinInput : InheritableAttr {
   let Documentation = [HLSLVkExtBuiltinInputDocs];
 }
 
+def HLSLVkConstantId : InheritableAttr {
+  let Spellings = [CXX11<"vk", "constant_id">];
+  let Args = [IntArgument<"Id">];
+  let Subjects = SubjectList<[ExternalGlobalVar]>;
+  let LangOpts = [HLSL];
+  let Documentation = [VkConstantIdDocs];
+}
+
 def RandomizeLayout : InheritableAttr {
   let Spellings = [GCC<"randomize_layout">];
   let Subjects = SubjectList<[Record]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 6051e1fc4511..43442f177ab7 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -8252,6 +8252,21 @@ and https://microsoft.github.io/hlsl-specs/proposals/0013-wave-size-range.html
   }];
 }
 
+def VkConstantIdDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``vk::constant_id`` attribute specifies the id for a SPIR-V specialization
+constant. The attribute applies to const global scalar variables. The variable must be initialized with a C++11 constexpr.
+In SPIR-V, the
+variable will be replaced with an `OpSpecConstant` with the given id.
+The syntax is:
+
+.. code-block:: text
+
+  ``[[vk::constant_id(<Id>)]] const T Name = <Init>``
+}];
+}
+
 def RootSignatureDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 68cd3d790e78..d65b3a5d2f44 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5065,6 +5065,19 @@ def HLSLGroupMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void()";
 }
 
+class HLSLScalarTemplate
+    : Template<["bool", "char", "short", "int", "long long int",
+                "unsigned short", "unsigned int", "unsigned long long int",
+                "__fp16", "float", "double"],
+               ["_bool", "_char", "_short", "_int", "_longlong", "_ushort",
+                "_uint", "_ulonglong", "_half", "_float", "_double"]>;
+
+def HLSLGetSpirvSpecConstant : LangBuiltin<"HLSL_LANG">, HLSLScalarTemplate {
+  let Spellings = ["__builtin_get_spirv_spec_constant"];
+  let Attributes = [NoThrow, Const, Pure];
+  let Prototype = "T(unsigned int, T)";
+}
+
 // Builtins for XRay.
 def XRayCustomEvent : Builtin {
   let Spellings = ["__xray_customevent"];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 979ff60b73b7..34b798a09c21 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12927,6 +12927,10 @@ def err_spirv_enum_not_int : Error<
 def err_spirv_enum_not_valid : Error<
    "invalid value for %select{storage class}0 argument">;
 
+def err_specialization_const
+    : Error<"variable with 'vk::constant_id' attribute must be a const "
+            "int/float/enum/bool and be initialized with a literal">;
+
 // errors of expect.with.probability
 def err_probability_not_constant_float : Error<
    "probability argument to __builtin_expect_with_probability must be constant "
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 33c4b8d1568b..97091792ba23 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -98,6 +98,8 @@ public:
   HLSLWaveSizeAttr *mergeWaveSizeAttr(Decl *D, const AttributeCommonInfo &AL,
                                       int Min, int Max, int Preferred,
                                       int SpelledArgsCount);
+  HLSLVkConstantIdAttr *
+  mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL, int Id);
   HLSLShaderAttr *mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                                   llvm::Triple::EnvironmentType ShaderType);
   HLSLParamModifierAttr *
@@ -135,6 +137,7 @@ public:
   void handleRootSignatureAttr(Decl *D, const ParsedAttr &AL);
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
   void handleWaveSizeAttr(Decl *D, const ParsedAttr &AL);
+  void handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL);
@@ -171,7 +174,7 @@ public:
   QualType getInoutParameterType(QualType Ty);
 
   bool transformInitList(const InitializedEntity &Entity, InitListExpr *Init);
-
+  bool handleInitialization(VarDecl *VDecl, Expr *&Init);
   void deduceAddressSpace(VarDecl *Decl);
 
 private:
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index ccf45c0c6ff1..cbc5ef9cb0d5 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -12,6 +12,7 @@
 
 #include "CGBuiltin.h"
 #include "CGHLSLRuntime.h"
+#include "CodeGenFunction.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -214,6 +215,43 @@ static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,
   }
 }
 
+// Returns the mangled name for a builtin function that the SPIR-V backend
+// will expand into a spec Constant.
+static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
+                                               ASTContext &Context) {
+  // The parameter types for our conceptual intrinsic function.
+  QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
+
+  // Create a temporary FunctionDecl for the builtin fuction. It won't be
+  // added to the AST.
+  FunctionProtoType::ExtProtoInfo EPI;
+  QualType FnType =
+      Context.getFunctionType(SpecConstantType, ClangParamTypes, EPI);
+  DeclarationName FuncName = &Context.Idents.get("__spirv_SpecConstant");
+  FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
+      Context, Context.getTranslationUnitDecl(), SourceLocation(),
+      SourceLocation(), FuncName, FnType, /*TSI=*/nullptr, SC_Extern);
+
+  // Attach the created parameter declarations to the function declaration.
+  SmallVector<ParmVarDecl *, 2> ParamDecls;
+  for (QualType ParamType : ClangParamTypes) {
+    ParmVarDecl *PD = ParmVarDecl::Create(
+        Context, FnDeclForMangling, SourceLocation(), SourceLocation(),
+        /*IdentifierInfo*/ nullptr, ParamType, /*TSI*/ nullptr, SC_None,
+        /*DefaultArg*/ nullptr);
+    ParamDecls.push_back(PD);
+  }
+  FnDeclForMangling->setParams(ParamDecls);
+
+  // Get the mangled name.
+  std::string Name;
+  llvm::raw_string_ostream MangledNameStream(Name);
+  MangleContext *Mangler = Context.createMangleContext();
+  Mangler->mangleName(FnDeclForMangling, MangledNameStream);
+  MangledNameStream.flush();
+  return Name;
+}
+
 Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
                                             const CallExpr *E,
                                             ReturnValueSlot ReturnValue) {
@@ -773,6 +811,42 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return EmitRuntimeCall(
         Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
   }
+  case Builtin::BI__builtin_get_spirv_spec_constant_bool:
+  case Builtin::BI__builtin_get_spirv_spec_constant_short:
+  case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
+  case Builtin::BI__builtin_get_spirv_spec_constant_int:
+  case Builtin::BI__builtin_get_spirv_spec_constant_uint:
+  case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
+  case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
+  case Builtin::BI__builtin_get_spirv_spec_constant_half:
+  case Builtin::BI__builtin_get_spirv_spec_constant_float:
+  case Builtin::BI__builtin_get_spirv_spec_constant_double: {
+    llvm::Function *SpecConstantFn = getSpecConstantFunction(E->getType());
+    llvm::Value *SpecId = EmitScalarExpr(E->getArg(0));
+    llvm::Value *DefaultVal = EmitScalarExpr(E->getArg(1));
+    llvm::Value *Args[] = {SpecId, DefaultVal};
+    return Builder.CreateCall(SpecConstantFn, Args);
+  }
   }
   return nullptr;
 }
+
+llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
+    const clang::QualType &SpecConstantType) {
+
+  // Find or create the declaration for the function.
+  llvm::Module *M = &CGM.getModule();
+  std::string MangledName =
+      getSpecConstantFunctionName(SpecConstantType, getContext());
+  llvm::Function *SpecConstantFn = M->getFunction(MangledName);
+
+  if (!SpecConstantFn) {
+    llvm::Type *IntType = ConvertType(getContext().IntTy);
+    llvm::Type *RetTy = ConvertType(SpecConstantType);
+    llvm::Type *ArgTypes[] = {IntType, RetTy};
+    llvm::FunctionType *FnTy = llvm::FunctionType::get(RetTy, ArgTypes, false);
+    SpecConstantFn = llvm::Function::Create(
+        FnTy, llvm::GlobalValue::ExternalLinkage, MangledName, M);
+  }
+  return SpecConstantFn;
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a5ab9df01dba..59f14b3e35fd 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4850,6 +4850,12 @@ public:
   llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
                                    ReturnValueSlot ReturnValue);
+
+  // Returns a builtin function that the SPIR-V backend will expand into a spec
+  // constant.
+  llvm::Function *
+  getSpecConstantFunction(const clang::QualType &SpecConstantType);
+
   llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 1bf72e5bb7b9..e1cccf068b5a 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2890,6 +2890,8 @@ static bool mergeDeclAttribute(Sema &S, NamedDecl *D,
     NewAttr = S.HLSL().mergeWaveSizeAttr(D, *WS, WS->getMin(), WS->getMax(),
                                          WS->getPreferred(),
                                          WS->getSpelledArgsCount());
+  else if (const auto *CI = dyn_cast<HLSLVkConstantIdAttr>(Attr))
+    NewAttr = S.HLSL().mergeVkConstantIdAttr(D, *CI, CI->getId());
   else if (const auto *SA = dyn_cast<HLSLShaderAttr>(Attr))
     NewAttr = S.HLSL().mergeShaderAttr(D, *SA, SA->getType());
   else if (isa<SuppressAttr>(Attr))
@@ -13757,6 +13759,10 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     return;
   }
 
+  if (getLangOpts().HLSL)
+    if (!HLSL().handleInitialization(VDecl, Init))
+      return;
+
   // Get the decls type and save a reference for later, since
   // CheckInitializerTypes may change it.
   QualType DclT = VDecl->getType(), SavT = DclT;
@@ -14179,6 +14185,13 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
       }
     }
 
+    // HLSL variable with the `vk::constant_id` attribute must be initialized.
+    if (!Var->isInvalidDecl() && Var->hasAttr<HLSLVkConstantIdAttr>()) {
+      Diag(Var->getLocation(), diag::err_specialization_const);
+      Var->setInvalidDecl();
+      return;
+    }
+
     if (!Var->isInvalidDecl() && RealDecl->hasAttr<LoaderUninitializedAttr>()) {
       if (Var->getStorageClass() == SC_Extern) {
         Diag(Var->getLocation(), diag::err_loader_uninitialized_extern_decl)
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 1c2fa80e782d..eba29e609cb0 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7590,6 +7590,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLVkExtBuiltinInput:
     S.HLSL().handleVkExtBuiltinInputAttr(D, AL);
     break;
+  case ParsedAttr::AT_HLSLVkConstantId:
+    S.HLSL().handleVkConstantIdAttr(D, AL);
+    break;
   case ParsedAttr::AT_HLSLSV_GroupThreadID:
     S.HLSL().handleSV_GroupThreadIDAttr(D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index b55f4fd786b5..9b43ee00810b 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -119,6 +119,40 @@ static ResourceClass getResourceClass(RegisterType RT) {
   llvm_unreachable("unexpected RegisterType value");
 }
 
+static Builtin::ID getSpecConstBuiltinId(QualType Type) {
+  const auto *BT = dyn_cast<BuiltinType>(Type);
+  if (!BT) {
+    if (!Type->isEnumeralType())
+      return Builtin::NotBuiltin;
+    return Builtin::BI__builtin_get_spirv_spec_constant_int;
+  }
+
+  switch (BT->getKind()) {
+  case BuiltinType::Bool:
+    return Builtin::BI__builtin_get_spirv_spec_constant_bool;
+  case BuiltinType::Short:
+    return Builtin::BI__builtin_get_spirv_spec_constant_short;
+  case BuiltinType::Int:
+    return Builtin::BI__builtin_get_spirv_spec_constant_int;
+  case BuiltinType::LongLong:
+    return Builtin::BI__builtin_get_spirv_spec_constant_longlong;
+  case BuiltinType::UShort:
+    return Builtin::BI__builtin_get_spirv_spec_constant_ushort;
+  case BuiltinType::UInt:
+    return Builtin::BI__builtin_get_spirv_spec_constant_uint;
+  case BuiltinType::ULongLong:
+    return Builtin::BI__builtin_get_spirv_spec_constant_ulonglong;
+  case BuiltinType::Half:
+    return Builtin::BI__builtin_get_spirv_spec_constant_half;
+  case BuiltinType::Float:
+    return Builtin::BI__builtin_get_spirv_spec_constant_float;
+  case BuiltinType::Double:
+    return Builtin::BI__builtin_get_spirv_spec_constant_double;
+  default:
+    return Builtin::NotBuiltin;
+  }
+}
+
 DeclBindingInfo *ResourceBindings::addDeclBindingInfo(const VarDecl *VD,
                                                       ResourceClass ResClass) {
   assert(getDeclBindingInfo(VD, ResClass) == nullptr &&
@@ -607,6 +641,41 @@ HLSLWaveSizeAttr *SemaHLSL::mergeWaveSizeAttr(Decl *D,
   return Result;
 }
 
+HLSLVkConstantIdAttr *
+SemaHLSL::mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL,
+                                int Id) {
+
+  auto &TargetInfo = getASTContext().getTargetInfo();
+  if (TargetInfo.getTriple().getArch() != llvm::Triple::spirv) {
+    Diag(AL.getLoc(), diag::warn_attribute_ignored) << AL;
+    return nullptr;
+  }
+
+  auto *VD = cast<VarDecl>(D);
+
+  if (getSpecConstBuiltinId(VD->getType()) == Builtin::NotBuiltin) {
+    Diag(VD->getLocation(), diag::err_specialization_const);
+    return nullptr;
+  }
+
+  if (!VD->getType().isConstQualified()) {
+    Diag(VD->getLocation(), diag::err_specialization_const);
+    return nullptr;
+  }
+
+  if (HLSLVkConstantIdAttr *CI = D->getAttr<HLSLVkConstantIdAttr>()) {
+    if (CI->getId() != Id) {
+      Diag(CI->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
+      Diag(AL.getLoc(), diag::note_conflicting_attribute);
+    }
+    return nullptr;
+  }
+
+  HLSLVkConstantIdAttr *Result =
+      ::new (getASTContext()) HLSLVkConstantIdAttr(getASTContext(), AL, Id);
+  return Result;
+}
+
 HLSLShaderAttr *
 SemaHLSL::mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                           llvm::Triple::EnvironmentType ShaderType) {
@@ -1157,6 +1226,15 @@ void SemaHLSL::handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL) {
                  HLSLVkExtBuiltinInputAttr(getASTContext(), AL, ID));
 }
 
+void SemaHLSL::handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL) {
+  uint32_t Id;
+  if (!SemaRef.checkUInt32Argument(AL, AL.getArgAsExpr(0), Id))
+    return;
+  HLSLVkConstantIdAttr *NewAttr = mergeVkConstantIdAttr(D, AL, Id);
+  if (NewAttr)
+    D->addAttr(NewAttr);
+}
+
 bool SemaHLSL::diagnoseInputIDType(QualType T, const ParsedAttr &AL) {
   const auto *VT = T->getAs<VectorType>();
 
@@ -3206,6 +3284,7 @@ static bool IsDefaultBufferConstantDecl(VarDecl *VD) {
   return VD->getDeclContext()->isTranslationUnit() &&
          QT.getAddressSpace() == LangAS::Default &&
          VD->getStorageClass() != SC_Static &&
+         !VD->hasAttr<HLSLVkConstantIdAttr>() &&
          !isInvalidConstantBufferLeafElementType(QT.getTypePtr());
 }
 
@@ -3273,7 +3352,8 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
     const Type *VarType = VD->getType().getTypePtr();
     while (VarType->isArrayType())
       VarType = VarType->getArrayElementTypeNoTypeQual();
-    if (VarType->isHLSLResourceRecord()) {
+    if (VarType->isHLSLResourceRecord() ||
+        VD->hasAttr<HLSLVkConstantIdAttr>()) {
       // Make the variable for resources static. The global externally visible
       // storage is accessed through the handle, which is a member. The variable
       // itself is not externally visible.
@@ -3696,3 +3776,41 @@ bool SemaHLSL::transformInitList(const InitializedEntity &Entity,
     Init->updateInit(Ctx, I, NewInit->getInit(I));
   return true;
 }
+
+bool SemaHLSL::handleInitialization(VarDecl *VDecl, Expr *&Init) {
+  const HLSLVkConstantIdAttr *ConstIdAttr =
+      VDecl->getAttr<HLSLVkConstantIdAttr>();
+  if (!ConstIdAttr)
+    return true;
+
+  ASTContext &Context = SemaRef.getASTContext();
+
+  APValue InitValue;
+  if (!Init->isCXX11ConstantExpr(Context, &InitValue)) {
+    Diag(VDecl->getLocation(), diag::err_specialization_const);
+    VDecl->setInvalidDecl();
+    return false;
+  }
+
+  Builtin::ID BID = getSpecConstBuiltinId(VDecl->getType());
+
+  // Argument 1: The ID from the attribute
+  int ConstantID = ConstIdAttr->getId();
+  llvm::APInt IDVal(Context.getIntWidth(Context.IntTy), ConstantID);
+  Expr *IdExpr = IntegerLiteral::Create(Context, IDVal, Context.IntTy,
+                                        ConstIdAttr->getLocation());
+
+  SmallVector<Expr *, 2> Args = {IdExpr, Init};
+  Expr *C = SemaRef.BuildBuiltinCallExpr(Init->getExprLoc(), BID, Args);
+  if (C->getType()->getCanonicalTypeUnqualified() !=
+      VDecl->getType()->getCanonicalTypeUnqualified()) {
+    C = SemaRef
+            .BuildCStyleCastExpr(SourceLocation(),
+                                 Context.getTrivialTypeSourceInfo(
+                                     Init->getType(), Init->getExprLoc()),
+                                 SourceLocation(), C)
+            .get();
+  }
+  Init = C;
+  return true;
+}
diff --git a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
new file mode 100644
index 000000000000..c0955c1ea7b4
--- /dev/null
+++ b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
@@ -0,0 +1,130 @@
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute -x hlsl -ast-dump -o - %s | FileCheck %s
+
+// CHECK: VarDecl {{.*}} bool_const 'const hlsl_private bool' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'bool'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool (*)(unsigned int, bool) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'bool (unsigned int, bool) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_bool' 'bool (unsigned int, bool) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 1
+// CHECK-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' true
+[[vk::constant_id(1)]]
+const bool bool_const = true;
+
+// CHECK: VarDecl {{.*}} short_const 'const hlsl_private short' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'short'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short (*)(unsigned int, short) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'short (unsigned int, short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_short' 'short (unsigned int, short) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 2
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
+[[vk::constant_id(2)]]
+const short short_const = 4;
+
+// CHECK: VarDecl {{.*}} int_const 'const hlsl_private int' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
+[[vk::constant_id(3)]]
+const int int_const = 5;
+
+// CHECK: VarDecl {{.*}} long_const 'const hlsl_private long long' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'long long'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long (*)(unsigned int, long long) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'long long (unsigned int, long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_longlong' 'long long (unsigned int, long long) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+[[vk::constant_id(4)]]
+const long long long_const = 8;
+
+// CHECK: VarDecl {{.*}} ushort_const 'const hlsl_private unsigned short' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned short'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short (*)(unsigned int, unsigned short) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned short (unsigned int, unsigned short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ushort' 'unsigned short (unsigned int, unsigned short) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10
+[[vk::constant_id(5)]]
+const unsigned short ushort_const = 10;
+
+// CHECK: VarDecl {{.*}} uint_const 'const hlsl_private unsigned int' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int (*)(unsigned int, unsigned int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int (unsigned int, unsigned int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_uint' 'unsigned int (unsigned int, unsigned int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 6
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12
+[[vk::constant_id(6)]]
+const unsigned int uint_const = 12;
+
+
+// CHECK: VarDecl {{.*}} ulong_const 'const hlsl_private unsigned long long' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned long long'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long (*)(unsigned int, unsigned long long) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned long long (unsigned int, unsigned long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ulonglong' 'unsigned long long (unsigned int, unsigned long long) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 7
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 25
+[[vk::constant_id(7)]]
+const unsigned long long ulong_const = 25;
+
+// CHECK: VarDecl {{.*}} half_const 'const hlsl_private half' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'half'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half (*)(unsigned int, half) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'half (unsigned int, half) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_half' 'half (unsigned int, half) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half' <FloatingCast>
+// CHECK-NEXT: FloatingLiteral {{.*}} 'float' 4.040000e+01
+[[vk::constant_id(8)]]
+const half half_const = 40.4;
+
+// CHECK: VarDecl {{.*}} float_const 'const hlsl_private float' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'float'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float (*)(unsigned int, float) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'float (unsigned int, float) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_float' 'float (unsigned int, float) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <IntegralToFloating>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 50
+[[vk::constant_id(8)]]
+const float float_const = 50;
+
+// CHECK: VarDecl {{.*}} double_const 'const hlsl_private double' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'double'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double (*)(unsigned int, double) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'double (unsigned int, double) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_double' 'double (unsigned int, double) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 9
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' <IntegralToFloating>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 100
+[[vk::constant_id(9)]]
+const double double_const = 100;
+
+// CHECK: VarDecl {{.*}} enum_const 'const hlsl_private E' static cinit
+// CHECK-NEXT: CStyleCastExpr {{.*}} 'E' <IntegralCast>
+// CHECK-NEXT: CallExpr {{.*}} 'int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10 
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int' <IntegralCast>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'E' EnumConstant {{.*}} 'e2' 'E' 
+enum E {
+    e0 = 10,
+    e1 = 20,
+    e2 = 30
+};
+
+[[vk::constant_id(10)]]
+const E enum_const = e2;
+
+// CHECK-NOT: CXXRecordDecl {{.*}} implicit struct __cblayout_$Globals definition
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
deleted file mode 100644
index 41cdd7d21bcb..000000000000
--- a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
+++ /dev/null
@@ -1,16 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
-
-// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
-struct S {
-    int a;
-    Int b;
-};
-
-[numthreads(1,1,1)]
-void main() {
-    S value;
-}
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
deleted file mode 100644
index 7149be0122f4..000000000000
--- a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
+++ /dev/null
@@ -1,68 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-template<class T, uint64_t Size>
-using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
-
-template<uint64_t Size>
-using ArrayBuffer = Array<RWBuffer<float>, Size>;
-
-typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
-
-typedef Array<Int, 5> ArrayInt;
-
-// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
-struct S {
-    ArrayBuffer<4> b;
-    Int i;
-};
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
-ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
-    return v;
-}
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
-Int getInt(Int v) {
-    return v;
-}
-
-// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
-// ArrayBuffer<4> g_buffers;
-// Int g_word;
-
-[numthreads(1, 1, 1)]
-void main() {
-    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    ArrayBuffer<4> buffers;
-
-    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
-    ArrayBuffer<0x123456789> longBuffers;
-
-    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-    Int word;
-
-    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
-    Int words[4];
-
-    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
-    ArrayInt words2;
-
-    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
-    S value;
-
-    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
-    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
-    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
-    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
-    Int word2 = getInt(word);
-}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
new file mode 100644
index 000000000000..41cdd7d21bcb
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
+
+// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
+struct S {
+    int a;
+    Int b;
+};
+
+[numthreads(1,1,1)]
+void main() {
+    S value;
+}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
new file mode 100644
index 000000000000..7149be0122f4
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+template<class T, uint64_t Size>
+using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
+
+template<uint64_t Size>
+using ArrayBuffer = Array<RWBuffer<float>, Size>;
+
+typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
+
+typedef Array<Int, 5> ArrayInt;
+
+// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
+struct S {
+    ArrayBuffer<4> b;
+    Int i;
+};
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
+ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
+    return v;
+}
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
+Int getInt(Int v) {
+    return v;
+}
+
+// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
+// ArrayBuffer<4> g_buffers;
+// Int g_word;
+
+[numthreads(1, 1, 1)]
+void main() {
+    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    ArrayBuffer<4> buffers;
+
+    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
+    ArrayBuffer<0x123456789> longBuffers;
+
+    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+    Int word;
+
+    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
+    Int words[4];
+
+    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
+    ArrayInt words2;
+
+    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
+    S value;
+
+    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
+    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
+    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
+    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
+    Int word2 = getInt(word);
+}
diff --git a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
new file mode 100644
index 000000000000..cbc1fa61eae2
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
@@ -0,0 +1,210 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 5
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+[[vk::constant_id(1)]]
+const bool bool_const = true;
+
+[[vk::constant_id(1)]]
+const short short_const = 4;
+
+[[vk::constant_id(3)]]
+const int int_const = 5;
+
+[[vk::constant_id(4)]]
+const long long long_const = 8;
+
+[[vk::constant_id(5)]]
+const unsigned short ushort_const = 10;
+
+[[vk::constant_id(6)]]
+const unsigned int uint_const = 12;
+
+[[vk::constant_id(7)]]
+const unsigned long long ulong_const = 25;
+
+[[vk::constant_id(8)]]
+const half half_const = 40.4;
+
+[[vk::constant_id(8)]]
+const float float_const = 50.5;
+
+[[vk::constant_id(9)]]
+const double double_const = 100.2;
+
+enum E {
+    e0 = 10,
+    e1 = 20,
+    e2 = 30
+};
+
+[[vk::constant_id(10)]]
+const E enum_const = e2;
+
+[numthreads(1,1,1)]
+void main() {
+    bool b = bool_const;
+    short s = short_const;
+    int i = int_const;
+    long long l = long_const;
+    unsigned short us = ushort_const;
+    unsigned int ui = uint_const;
+    unsigned long long ul = ulong_const;
+    half h = half_const;
+    float f = float_const;
+    double d = double_const;
+    E e = enum_const;
+}
+//.
+// CHECK: @_ZL10bool_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL11short_const = internal addrspace(10) global i16 0, align 2
+// CHECK: @_ZL9int_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL10long_const = internal addrspace(10) global i64 0, align 8
+// CHECK: @_ZL12ushort_const = internal addrspace(10) global i16 0, align 2
+// CHECK: @_ZL10uint_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL11ulong_const = internal addrspace(10) global i64 0, align 8
+// CHECK: @_ZL10half_const = internal addrspace(10) global float 0.000000e+00, align 4
+// CHECK: @_ZL11float_const = internal addrspace(10) global float 0.000000e+00, align 4
+// CHECK: @_ZL12double_const = internal addrspace(10) global double 0.000000e+00, align 8
+// CHECK: @_ZL10enum_const = internal addrspace(10) global i32 0, align 4
+//.
+// CHECK-LABEL: define internal spir_func void @_Z4mainv(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[S:%.*]] = alloca i16, align 2
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[L:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[US:%.*]] = alloca i16, align 2
+// CHECK-NEXT:    [[UI:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[UL:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[H:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[F:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[D:%.*]] = alloca double, align 8
+// CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(10) @_ZL10bool_const, align 4
+// CHECK-NEXT:    [[LOADEDV:%.*]] = trunc i32 [[TMP1]] to i1
+// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[LOADEDV]] to i32
+// CHECK-NEXT:    store i32 [[STOREDV]], ptr [[B]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(10) @_ZL11short_const, align 2
+// CHECK-NEXT:    store i16 [[TMP2]], ptr [[S]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(10) @_ZL9int_const, align 4
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr addrspace(10) @_ZL10long_const, align 8
+// CHECK-NEXT:    store i64 [[TMP4]], ptr [[L]], align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(10) @_ZL12ushort_const, align 2
+// CHECK-NEXT:    store i16 [[TMP5]], ptr [[US]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(10) @_ZL10uint_const, align 4
+// CHECK-NEXT:    store i32 [[TMP6]], ptr [[UI]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr addrspace(10) @_ZL11ulong_const, align 8
+// CHECK-NEXT:    store i64 [[TMP7]], ptr [[UL]], align 8
+// CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr addrspace(10) @_ZL10half_const, align 4
+// CHECK-NEXT:    store float [[TMP8]], ptr [[H]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr addrspace(10) @_ZL11float_const, align 4
+// CHECK-NEXT:    store float [[TMP9]], ptr [[F]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr addrspace(10) @_ZL12double_const, align 8
+// CHECK-NEXT:    store double [[TMP10]], ptr [[D]], align 8
+// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr addrspace(10) @_ZL10enum_const, align 4
+// CHECK-NEXT:    store i32 [[TMP11]], ptr [[E]], align 4
+// CHECK-NEXT:    ret void
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init(
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i1 @_Z20__spirv_SpecConstantib(i32 1, i1 true)
+// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[TMP1]] to i32
+// CHECK-NEXT:    store i32 [[STOREDV]], ptr addrspace(10) @_ZL10bool_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.1(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantis(i32 1, i16 4)
+// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL11short_const, align 2
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.2(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 3, i32 5)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL9int_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.3(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantix(i32 4, i64 8)
+// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL10long_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.4(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantit(i32 5, i16 10)
+// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL12ushort_const, align 2
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.5(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantij(i32 6, i32 12)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10uint_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.6(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantiy(i32 7, i64 25)
+// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL11ulong_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.7(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantiDh(i32 8, float 0x4044333340000000)
+// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL10half_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.8(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantif(i32 8, float 5.050000e+01)
+// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL11float_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.9(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn double @_Z20__spirv_SpecConstantid(i32 9, double 0x40590CCCC0000000)
+// CHECK-NEXT:    store double [[TMP1]], ptr addrspace(10) @_ZL12double_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.10(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 10, i32 30)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10enum_const, align 4
+// CHECK-NEXT:    ret void
diff --git a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
new file mode 100644
index 000000000000..24873d272a54
--- /dev/null
+++ b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan1.3-compute -verify %s
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.8-compute -verify %s
+
+#ifndef __spirv__
+// expected-warning@+2{{'constant_id' attribute ignored}}
+#endif
+[[vk::constant_id(0)]]
+const bool sc0 = true;
+
+#ifdef __spirv__
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(1)]]
+const bool sc1 = sc0; // error
+
+// expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+[[vk::constant_id(2)]]
+static const bool sc2 = false; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(3)]]
+const bool sc3; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(4)]]
+bool sc4 = false; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(5)]]
+const int2 sc5 = {0,0}; // error
+
+[numthreads(1,1,1)]
+void main() {
+  // expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+  [[vk::constant_id(6)]]
+  const bool sc6 = false; // error
+}
+#endif
-- 
cgit v1.2.3


From d3441f7348203cc2a1d9c44fd24c1113954aa2b2 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Wed, 18 Jun 2025 11:45:20 +0100
Subject: [LV] Change getSmallBestKnownTC to return an ElementCount (NFC)
 (#141793)

This is prep work for enabling better UF calculations when using vscale
based VFs to vectorise loops with vscale based tripcounts.

NOTE: NFC because All uses remain fixed-length until a following PR
changes LoopVectorize's version of getSmallConstantTripCount().
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 50 +++++++++++++++----------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 16d48b06dce4..2f4416d2782e 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -419,6 +419,13 @@ static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
   return DL.getTypeAllocSizeInBits(Ty) != DL.getTypeSizeInBits(Ty);
 }
 
+/// A version of ScalarEvolution::getSmallConstantTripCount that returns an
+/// ElementCount to include loops whose trip count is a function of vscale.
+static ElementCount getSmallConstantTripCount(ScalarEvolution *SE,
+                                              const Loop *L) {
+  return ElementCount::getFixed(SE->getSmallConstantTripCount(L));
+}
+
 /// Returns "best known" trip count, which is either a valid positive trip count
 /// or std::nullopt when an estimate cannot be made (including when the trip
 /// count would overflow), for the specified loop \p L as defined by the
@@ -427,24 +434,24 @@ static bool hasIrregularType(Type *Ty, const DataLayout &DL) {
 ///   2) Returns expected trip count according to profile data if any.
 ///   3) Returns upper bound estimate if known, and if \p CanUseConstantMax.
 ///   4) Returns std::nullopt if all of the above failed.
-static std::optional<unsigned>
+static std::optional<ElementCount>
 getSmallBestKnownTC(PredicatedScalarEvolution &PSE, Loop *L,
                     bool CanUseConstantMax = true) {
   // Check if exact trip count is known.
-  if (unsigned ExpectedTC = PSE.getSE()->getSmallConstantTripCount(L))
+  if (auto ExpectedTC = getSmallConstantTripCount(PSE.getSE(), L))
     return ExpectedTC;
 
   // Check if there is an expected trip count available from profile data.
   if (LoopVectorizeWithBlockFrequency)
     if (auto EstimatedTC = getLoopEstimatedTripCount(L))
-      return *EstimatedTC;
+      return ElementCount::getFixed(*EstimatedTC);
 
   if (!CanUseConstantMax)
     return std::nullopt;
 
   // Check if upper bound estimate is known.
   if (unsigned ExpectedTC = PSE.getSmallConstantMaxTripCount())
-    return ExpectedTC;
+    return ElementCount::getFixed(ExpectedTC);
 
   return std::nullopt;
 }
@@ -1960,7 +1967,8 @@ public:
           // Get the best known TC estimate.
           if (auto EstimatedTC = getSmallBestKnownTC(
                   PSE, OuterLoop, /* CanUseConstantMax = */ false))
-            BestTripCount = *EstimatedTC;
+            if (EstimatedTC->isFixed())
+              BestTripCount = EstimatedTC->getFixedValue();
 
           InstructionCost NewMemCheckCost = MemCheckCost / BestTripCount;
 
@@ -3750,12 +3758,12 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   }
 
   ScalarEvolution *SE = PSE.getSE();
-  unsigned TC = SE->getSmallConstantTripCount(TheLoop);
+  ElementCount TC = getSmallConstantTripCount(SE, TheLoop);
   unsigned MaxTC = PSE.getSmallConstantMaxTripCount();
   LLVM_DEBUG(dbgs() << "LV: Found trip count: " << TC << '\n');
-  if (TC != MaxTC)
+  if (TC != ElementCount::getFixed(MaxTC))
     LLVM_DEBUG(dbgs() << "LV: Found maximum trip count: " << MaxTC << '\n');
-  if (TC == 1) {
+  if (TC.isScalar()) {
     reportVectorizationFailure("Single iteration (non) loop",
         "loop trip count is one, irrelevant for vectorization",
         "SingleIterationLoop", ORE, TheLoop);
@@ -3869,7 +3877,9 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   }
 
   auto ExpectedTC = getSmallBestKnownTC(PSE, TheLoop);
-  if (ExpectedTC && ExpectedTC <= TTI.getMinTripCountTailFoldingThreshold()) {
+  if (ExpectedTC && ExpectedTC->isFixed() &&
+      ExpectedTC->getFixedValue() <=
+          TTI.getMinTripCountTailFoldingThreshold()) {
     if (MaxPowerOf2RuntimeVF > 0u) {
       // If we have a low-trip-count, and the fixed-width VF is known to divide
       // the trip count but the scalable factor does not, use the fixed-width
@@ -3927,7 +3937,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
     return FixedScalableVFPair::getNone();
   }
 
-  if (TC == 0) {
+  if (TC.isZero()) {
     reportVectorizationFailure(
         "unable to calculate the loop count due to complex control flow",
         "UnknownLoopCountComplexCFG", ORE, TheLoop);
@@ -4816,13 +4826,13 @@ LoopVectorizationCostModel::selectInterleaveCount(VPlan &Plan, ElementCount VF,
     // At least one iteration must be scalar when this constraint holds. So the
     // maximum available iterations for interleaving is one less.
     unsigned AvailableTC = requiresScalarEpilogue(VF.isVector())
-                               ? (*BestKnownTC) - 1
-                               : *BestKnownTC;
+                               ? BestKnownTC->getFixedValue() - 1
+                               : BestKnownTC->getFixedValue();
 
     unsigned InterleaveCountLB = bit_floor(std::max(
         1u, std::min(AvailableTC / (EstimatedVF * 2), MaxInterleaveCount)));
 
-    if (PSE.getSE()->getSmallConstantTripCount(TheLoop) > 0) {
+    if (getSmallConstantTripCount(PSE.getSE(), TheLoop).isNonZero()) {
       // If the best known trip count is exact, we select between two
       // prospective ICs, where
       //
@@ -5182,8 +5192,8 @@ InstructionCost LoopVectorizationCostModel::expectedCost(ElementCount VF) {
   // costs of comparison and induction instructions, as they'll get simplified
   // away.
   SmallPtrSet<Instruction *, 2> ValuesToIgnoreForVF;
-  auto TC = PSE.getSE()->getSmallConstantTripCount(TheLoop);
-  if (VF.isFixed() && TC == VF.getFixedValue() && !foldTailByMasking())
+  auto TC = getSmallConstantTripCount(PSE.getSE(), TheLoop);
+  if (TC == VF && !foldTailByMasking())
     addFullyUnrolledInstructionsToIgnore(TheLoop, Legal->getInductionVars(),
                                          ValuesToIgnoreForVF);
 
@@ -6878,8 +6888,8 @@ LoopVectorizationPlanner::precomputeCosts(VPlan &Plan, ElementCount VF,
     // simplified away.
     // TODO: Remove this code after stepping away from the legacy cost model and
     // adding code to simplify VPlans before calculating their costs.
-    auto TC = PSE.getSE()->getSmallConstantTripCount(OrigLoop);
-    if (VF.isFixed() && TC == VF.getFixedValue() && !CM.foldTailByMasking())
+    auto TC = getSmallConstantTripCount(PSE.getSE(), OrigLoop);
+    if (TC == VF && !CM.foldTailByMasking())
       addFullyUnrolledInstructionsToIgnore(OrigLoop, Legal->getInductionVars(),
                                            CostCtx.SkipCostComputation);
 
@@ -9647,8 +9657,7 @@ static bool isOutsideLoopWorkProfitable(GeneratedRTChecks &Checks,
   // Skip vectorization if the expected trip count is less than the minimum
   // required trip count.
   if (auto ExpectedTC = getSmallBestKnownTC(PSE, L)) {
-    if (ElementCount::isKnownLT(ElementCount::getFixed(*ExpectedTC),
-                                VF.MinProfitableTripCount)) {
+    if (ElementCount::isKnownLT(*ExpectedTC, VF.MinProfitableTripCount)) {
       LLVM_DEBUG(dbgs() << "LV: Vectorization is not beneficial: expected "
                            "trip count < minimum profitable VF ("
                         << *ExpectedTC << " < " << VF.MinProfitableTripCount
@@ -10018,7 +10027,8 @@ bool LoopVectorizePass::processLoop(Loop *L) {
   // Check the loop for a trip count threshold: vectorize loops with a tiny trip
   // count by optimizing for size, to minimize overheads.
   auto ExpectedTC = getSmallBestKnownTC(PSE, L);
-  if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
+  if (ExpectedTC && ExpectedTC->isFixed() &&
+      ExpectedTC->getFixedValue() < TinyTripCountVectorThreshold) {
     LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
                       << "This loop is worth vectorizing only if no scalar "
                       << "iteration overheads are incurred.");
-- 
cgit v1.2.3


From b7ef5dbac91f9ccaf335ae4dd998e5783523f24e Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Wed, 18 Jun 2025 11:53:27 +0100
Subject: [LLVM][ComplexDeinterleaving] Update splat identification to include
 vector ConstantInt/FP. (#144516)

---
 llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp                     | 3 +++
 llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll | 1 +
 2 files changed, 4 insertions(+)

diff --git a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
index ae12423d827d..8855740f0cc8 100644
--- a/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
+++ b/llvm/lib/CodeGen/ComplexDeinterleavingPass.cpp
@@ -2005,6 +2005,9 @@ ComplexDeinterleavingGraph::identifySplat(Value *R, Value *I) {
     if (isa<ConstantDataVector>(V))
       return true;
 
+    if (isa<ConstantInt>(V) || isa<ConstantFP>(V))
+      return isa<VectorType>(V->getType());
+
     VectorType *VTy;
     ArrayRef<int> Mask;
     // Splats are represented differently depending on whether the repeated
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll
index 661531361315..e7a00fc90e31 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat-scalable.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s --mattr=+sve -o - | FileCheck %s
+; RUN: llc -use-constant-int-for-scalable-splat -use-constant-fp-for-scalable-splat < %s --mattr=+sve -o - | FileCheck %s
 
 target triple = "aarch64"
 
-- 
cgit v1.2.3


From b5967264b0fbfd502b3a7edec27409e966fb68be Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 18 Jun 2025 19:56:26 +0900
Subject: CodeGen: Move ABI option enums to support (#142912)

Move these out of TargetOptions and into Support to avoid
the dependency on Target. There are similar ABI options
already in Support/CodeGen.h.
---
 llvm/include/llvm/Support/CodeGen.h      | 16 ++++++++++++++++
 llvm/include/llvm/Target/TargetOptions.h | 17 +----------------
 2 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h
index 48745f7f4d2a..90733b50385a 100644
--- a/llvm/include/llvm/Support/CodeGen.h
+++ b/llvm/include/llvm/Support/CodeGen.h
@@ -50,6 +50,22 @@ namespace llvm {
     };
   }
 
+  namespace FloatABI {
+  enum ABIType {
+    Default, // Target-specific (either soft or hard depending on triple, etc).
+    Soft,    // Soft float.
+    Hard     // Hard float.
+  };
+  }
+
+  enum class EABI {
+    Unknown,
+    Default, // Default means not specified
+    EABI4,   // Target-specific (either 4, 5 or gnu depending on triple).
+    EABI5,
+    GNU
+  };
+
   /// Code generation optimization level.
   enum class CodeGenOptLevel {
     None = 0,      ///< -O0
diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h
index a7c46921255b..f420798aa46f 100644
--- a/llvm/include/llvm/Target/TargetOptions.h
+++ b/llvm/include/llvm/Target/TargetOptions.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/FloatingPointMode.h"
 #include "llvm/MC/MCTargetOptions.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
 
 #include <memory>
@@ -25,14 +26,6 @@ struct fltSemantics;
 class MachineFunction;
 class MemoryBuffer;
 
-namespace FloatABI {
-enum ABIType {
-  Default, // Target-specific (either soft or hard depending on triple, etc).
-  Soft,    // Soft float.
-  Hard     // Hard float.
-};
-}
-
 namespace FPOpFusion {
 enum FPOpFusionMode {
   Fast,     // Enable fusion of FP ops wherever it's profitable.
@@ -71,14 +64,6 @@ enum class BasicBlockSection {
   None    // Do not use Basic Block Sections.
 };
 
-enum class EABI {
-  Unknown,
-  Default, // Default means not specified
-  EABI4,   // Target-specific (either 4, 5 or gnu depending on triple).
-  EABI5,
-  GNU
-};
-
 /// Identify a debugger for "tuning" the debug info.
 ///
 /// The "debugger tuning" concept allows us to present a more intuitive
-- 
cgit v1.2.3


From 4aca3dc48b0919b81bd86302b141f29869266c45 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler@amd.com>
Date: Wed, 18 Jun 2025 13:04:24 +0200
Subject: Reland: [GlobalISel] prevent G_UNMERGE_VALUES for vectors with
 different elements (#144661)

This commit prevents building a G_UNMERGE_VALUES instruction with
different source and destination vector elements in
`LegalizationArtifactCombiner::ArtifactValueFinder::tryCombineMergeLike()`,
e.g.:
`%1:_(<2 x s8>), %2:_(<2 x s8>) = G_UNMERGE_VALUES %0:_(<2 x s16>)`

This LLVM defect was identified via the AMD Fuzzing project.
---
 .../GlobalISel/LegalizationArtifactCombiner.h      |  5 ++-
 .../CodeGen/AMDGPU/GlobalISel/insertelement.ll     | 44 ++++++++++++++++++++++
 .../buffer-fat-pointers-contents-legalization.ll   | 10 ++---
 3 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
index 22f6a5fde546..8f560c42082f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h
@@ -997,6 +997,7 @@ public:
 
       // Recognize UnmergeSrc that can be unmerged to DstTy directly.
       // Types have to be either both vector or both non-vector types.
+      // In case of vector types, the scalar elements need to match.
       // Merge-like opcodes are combined one at the time. First one creates new
       // unmerge, following should use the same unmerge (builder performs CSE).
       //
@@ -1005,7 +1006,9 @@ public:
       // %AnotherDst:_(DstTy) = G_merge_like_opcode %2:_(EltTy), %3
       //
       // %Dst:_(DstTy), %AnotherDst = G_UNMERGE_VALUES %UnmergeSrc
-      if ((DstTy.isVector() == UnmergeSrcTy.isVector()) &&
+      if (((!DstTy.isVector() && !UnmergeSrcTy.isVector()) ||
+           (DstTy.isVector() && UnmergeSrcTy.isVector() &&
+            DstTy.getScalarType() == UnmergeSrcTy.getScalarType())) &&
           (Elt0UnmergeIdx % NumMIElts == 0) &&
           getCoverTy(UnmergeSrcTy, DstTy) == UnmergeSrcTy) {
         if (!isSequenceFromUnmerge(MI, 0, Unmerge, Elt0UnmergeIdx, NumMIElts,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index 8134eb3ca2af..51d0b225b2a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -6506,3 +6506,47 @@ entry:
   %insert = insertelement <5 x double> %vec, double %val, i32 %idx
   ret <5 x double> %insert
 }
+
+; Found by fuzzer, reduced with llvm-reduce.
+define void @insert_very_small_from_very_large(<32 x i16> %L3, ptr %ptr) {
+; GPRIDX-LABEL: insert_very_small_from_very_large:
+; GPRIDX:       ; %bb.0: ; %bb
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GPRIDX-NEXT:    v_and_b32_e32 v0, 1, v0
+; GPRIDX-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GPRIDX-NEXT:    v_and_b32_e32 v0, 3, v0
+; GPRIDX-NEXT:    flat_store_byte v[16:17], v0
+; GPRIDX-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GPRIDX-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: insert_very_small_from_very_large:
+; GFX10:       ; %bb.0: ; %bb
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_lshlrev_b16 v0, 1, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 3, v0
+; GFX10-NEXT:    flat_store_byte v[16:17], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: insert_very_small_from_very_large:
+; GFX11:       ; %bb.0: ; %bb
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    v_lshrrev_b16 v0.l, 1, v0.l
+; GFX11-NEXT:    v_and_b16 v0.l, v0.l, 1
+; GFX11-NEXT:    v_lshlrev_b16 v0.l, 1, v0.l
+; GFX11-NEXT:    v_and_b32_e32 v0, 3, v0
+; GFX11-NEXT:    flat_store_b8 v[16:17], v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
+bb:
+  %a = bitcast <32 x i16> %L3 to i512
+  %b = trunc i512 %a to i8
+  %c = trunc i8 %b to i2
+  %d = bitcast i2 %c to <2 x i1>
+  %insert = insertelement <2 x i1> %d, i1 false, i32 0
+  store <2 x i1> %insert, ptr %ptr, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index fdc1dd6cce8e..53b2542cf9a7 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2166,14 +2166,14 @@ define <6 x i8> @load_v6i8(ptr addrspace(8) inreg %buf) {
 ; GISEL-LABEL: load_v6i8:
 ; GISEL:       ; %bb.0:
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT:    buffer_load_ushort v4, off, s[16:19], 0 offset:4
 ; GISEL-NEXT:    buffer_load_dword v0, off, s[16:19], 0
+; GISEL-NEXT:    buffer_load_ushort v4, off, s[16:19], 0 offset:4
 ; GISEL-NEXT:    s_waitcnt vmcnt(1)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
-; GISEL-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
   %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
   %ret = load <6 x i8>, ptr addrspace(7) %p
@@ -3630,10 +3630,10 @@ define <6 x i8> @volatile_load_v6i8(ptr addrspace(8) inreg %buf) {
 ; GISEL-NEXT:    buffer_load_ushort v4, off, s[16:19], 0 offset:4 glc
 ; GISEL-NEXT:    s_waitcnt vmcnt(1)
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
-; GISEL-NEXT:    s_waitcnt vmcnt(0)
-; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v0
 ; GISEL-NEXT:    v_lshrrev_b32_e32 v3, 24, v0
+; GISEL-NEXT:    s_waitcnt vmcnt(0)
+; GISEL-NEXT:    v_lshrrev_b32_e32 v5, 8, v4
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
   %p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
   %ret = load volatile <6 x i8>, ptr addrspace(7) %p
-- 
cgit v1.2.3


From c3efe7d64cebcd8679bec3ba7ff8154f8b0a1fa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Storsj=C3=B6?= <martin@martin.st>
Date: Wed, 18 Jun 2025 14:12:14 +0300
Subject: [libcxx] [test] Fix odr_signature tests with optimizations enabled
 (#144317)

If optimization is enabled, the inline `f()` function actually gets
inlined, meaning that the functions `tu1()` and `tu2()` trivially return
1 and 2, instead of actually referencing the potentially linker
deduplicated function `f()`, which is what the test tries to test.

Therefore, this test previously actually failed to test what it was
supposed to test, if optimization was enabled.

Mark the inline functions with `TEST_NOINLINE` to make sure that they
don't get inlined even with optimizations enabled.

Also update the TODO comments to explain why we have an XFAIL for msvc
mode here.

This avoids these tests unexpectedly passing if building in msvc mode,
with optimizations enabled
(`-DLIBCXX_TEST_PARAMS="optimization=speed"`).
---
 libcxx/test/libcxx/odr_signature.exceptions.sh.cpp | 10 +++++++---
 libcxx/test/libcxx/odr_signature.hardening.sh.cpp  | 14 +++++++++-----
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/libcxx/test/libcxx/odr_signature.exceptions.sh.cpp b/libcxx/test/libcxx/odr_signature.exceptions.sh.cpp
index 6bf60b5e82d3..c0ba48eb245d 100644
--- a/libcxx/test/libcxx/odr_signature.exceptions.sh.cpp
+++ b/libcxx/test/libcxx/odr_signature.exceptions.sh.cpp
@@ -6,9 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-// TODO: Investigate
+// ABI tags have no effect in MSVC mode.
 // XFAIL: msvc
 
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 // Test that we encode whether exceptions are supported in an ABI tag to avoid
 // ODR violations when linking TUs that have different values for it.
 
@@ -18,17 +20,19 @@
 // RUN: %{cxx} %t.tu1.o %t.tu2.o %t.main.o %{flags} %{link_flags} -o %t.exe
 // RUN: %{exec} %t.exe
 
+#include "test_macros.h"
+
 // -fno-exceptions
 #ifdef TU1
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 1; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 1; }
 int tu1() { return f(); }
 #endif // TU1
 
 // -fexceptions
 #ifdef TU2
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 2; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 2; }
 int tu2() { return f(); }
 #endif // TU2
 
diff --git a/libcxx/test/libcxx/odr_signature.hardening.sh.cpp b/libcxx/test/libcxx/odr_signature.hardening.sh.cpp
index 0dc280bf2818..8daf3f3fd046 100644
--- a/libcxx/test/libcxx/odr_signature.hardening.sh.cpp
+++ b/libcxx/test/libcxx/odr_signature.hardening.sh.cpp
@@ -6,9 +6,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-// TODO: Investigate
+// ABI tags have no effect in MSVC mode.
 // XFAIL: msvc
 
+// XFAIL: FROZEN-CXX03-HEADERS-FIXME
+
 // Test that we encode the hardening mode in an ABI tag to avoid ODR violations
 // when linking TUs that have different values for it.
 
@@ -21,31 +23,33 @@
 // RUN: %{cxx} %t.tu1.o %t.tu2.o %t.tu3.o %t.tu4.o %t.main.o %{flags} %{link_flags} -o %t.exe
 // RUN: %{exec} %t.exe
 
+#include "test_macros.h"
+
 // fast hardening mode
 #ifdef TU1
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 1; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 1; }
 int tu1() { return f(); }
 #endif // TU1
 
 // extensive hardening mode
 #ifdef TU2
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 2; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 2; }
 int tu2() { return f(); }
 #endif // TU2
 
 // debug hardening mode
 #ifdef TU3
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 3; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 3; }
 int tu3() { return f(); }
 #endif // TU3
 
 // No hardening
 #ifdef TU4
 #  include <__config>
-_LIBCPP_HIDE_FROM_ABI inline int f() { return 4; }
+_LIBCPP_HIDE_FROM_ABI TEST_NOINLINE inline int f() { return 4; }
 int tu4() { return f(); }
 #endif // TU4
 
-- 
cgit v1.2.3


From 66d6964a55014e7fabb7c80fbba19d2145262b6b Mon Sep 17 00:00:00 2001
From: Garvit Gupta <quic_garvgupt@quicinc.com>
Date: Wed, 18 Jun 2025 16:50:48 +0530
Subject: Fix tests failing on fuchsia clang x86_64 builders (#144655)

Fuchsia sets CLANG_DEFAULT_UNWINDLIB to libunwind. As a result, when
rtlib is set to libgcc and unwindlib is not explicitly specified, tests
using Fuchsia as the default platform will fail. To address this, the
affected tests are now xfailed

This change fixes the following tests introduced in
https://github.com/llvm/llvm-project/commit/45ea46c44636094e9fcdbbeabfd11f9d0fad5e38:

clang/test/Driver/aarch64-toolchain-extra.c
clang/test/Driver/arm-toolchain-extra.c
clang/test/Driver/aarch64-toolchain.c
clang/test/Driver/arm-toolchain.c
---
 clang/test/Driver/aarch64-toolchain-extra.c | 1 +
 clang/test/Driver/aarch64-toolchain.c       | 1 +
 clang/test/Driver/arm-toolchain-extra.c     | 1 +
 clang/test/Driver/arm-toolchain.c           | 1 +
 4 files changed, 4 insertions(+)

diff --git a/clang/test/Driver/aarch64-toolchain-extra.c b/clang/test/Driver/aarch64-toolchain-extra.c
index 2610e962bd69..eb8c741ae1ad 100644
--- a/clang/test/Driver/aarch64-toolchain-extra.c
+++ b/clang/test/Driver/aarch64-toolchain-extra.c
@@ -3,6 +3,7 @@
 // The tests here are similar to those in aarch64-toolchain.c, however
 // these tests need to create symlinks to test directory trees in order to
 // set up the environment and therefore shell support is required.
+// XFAIL: target={{.*}}-fuchsia{{.*}}
 // REQUIRES: shell
 // UNSUPPORTED: system-windows
 
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
index 7f2c01d928e4..74841eec598b 100644
--- a/clang/test/Driver/aarch64-toolchain.c
+++ b/clang/test/Driver/aarch64-toolchain.c
@@ -1,3 +1,4 @@
+// XFAIL: target={{.*}}-fuchsia{{.*}}
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### %s -fuse-ld= \
diff --git a/clang/test/Driver/arm-toolchain-extra.c b/clang/test/Driver/arm-toolchain-extra.c
index 114de0a8154a..67206818f211 100644
--- a/clang/test/Driver/arm-toolchain-extra.c
+++ b/clang/test/Driver/arm-toolchain-extra.c
@@ -3,6 +3,7 @@
 // The tests here are similar to those in arm-toolchain.c, however
 // these tests need to create symlinks to test directory trees in order to
 // set up the environment and therefore shell support is required.
+// XFAIL: target={{.*}}-fuchsia{{.*}}
 // REQUIRES: shell
 // UNSUPPORTED: system-windows
 
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
index 2e38461fb7a3..56a0e0de7ba7 100644
--- a/clang/test/Driver/arm-toolchain.c
+++ b/clang/test/Driver/arm-toolchain.c
@@ -1,3 +1,4 @@
+// XFAIL: target={{.*}}-fuchsia{{.*}}
 // UNSUPPORTED: system-windows
 
 // RUN: %clang -### %s -fuse-ld= \
-- 
cgit v1.2.3


From 8a469da8b2342dd9104faf25deeddd8ad66ca6a6 Mon Sep 17 00:00:00 2001
From: "Oleksandr \"Alex\" Zinenko" <git@ozinenko.com>
Date: Wed, 18 Jun 2025 13:32:46 +0200
Subject: [mlir] remove unnecessary atomic_rmw expansions (#144515)

The expansion of `memref.atomic_rmw` into a `memref.generic_atomic_rmw`
for floating-point min/max operations is no longer necessary as those
are now supported by the LLVM dialect and LLVM IR.

Furthermore, combining this expansion with direct lowering of
`generic_atomic_rmw` could leads to invalid LLVM dialect IR with
`cmpxchg` operating on floating-point values that it does not support.
---
 mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp | 56 +-----------------------
 mlir/test/Dialect/MemRef/expand-ops.mlir         | 38 ++--------------
 2 files changed, 5 insertions(+), 89 deletions(-)

diff --git a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
index 020aabd9db6d..a617029ce470 100644
--- a/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/MemRef/Transforms/ExpandOps.cpp
@@ -1,16 +1,10 @@
-//===- StdExpandDivs.cpp - Code to prepare Std for lowering Divs to LLVM  -===//
+//===- ExpandDivs.cpp - Expansion patterns for MemRef operations ----------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// This file Std transformations to expand Divs operation to help for the
-// lowering to LLVM. Currently implemented transformations are Ceil and Floor
-// for Signed Integers.
-//
-//===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/MemRef/Transforms/Passes.h"
 
@@ -33,44 +27,6 @@ using namespace mlir;
 
 namespace {
 
-/// Converts `atomic_rmw` that cannot be lowered to a simple atomic op with
-/// AtomicRMWOpLowering pattern, such as minimum and maximum operations for
-/// floating-point numbers, to `memref.generic_atomic_rmw` with the expanded
-/// code.
-///
-/// %x = atomic_rmw maximumf %fval, %F[%i] : (f32, memref<10xf32>) -> f32
-///
-/// will be lowered to
-///
-/// %x = memref.generic_atomic_rmw %F[%i] : memref<10xf32> {
-/// ^bb0(%current: f32):
-///   %1 = arith.maximumf %current, %fval : f32
-///   memref.atomic_yield %1 : f32
-/// }
-struct AtomicRMWOpConverter : public OpRewritePattern<memref::AtomicRMWOp> {
-public:
-  using OpRewritePattern::OpRewritePattern;
-
-  LogicalResult matchAndRewrite(memref::AtomicRMWOp op,
-                                PatternRewriter &rewriter) const final {
-    auto loc = op.getLoc();
-    auto genericOp = rewriter.create<memref::GenericAtomicRMWOp>(
-        loc, op.getMemref(), op.getIndices());
-    OpBuilder bodyBuilder =
-        OpBuilder::atBlockEnd(genericOp.getBody(), rewriter.getListener());
-
-    Value lhs = genericOp.getCurrentValue();
-    Value rhs = op.getValue();
-
-    Value arithOp =
-        mlir::arith::getReductionOp(op.getKind(), bodyBuilder, loc, lhs, rhs);
-    bodyBuilder.create<memref::AtomicYieldOp>(loc, arithOp);
-
-    rewriter.replaceOp(op, genericOp.getResult());
-    return success();
-  }
-};
-
 /// Converts `memref.reshape` that has a target shape of a statically-known
 /// size to `memref.reinterpret_cast`.
 struct MemRefReshapeOpConverter : public OpRewritePattern<memref::ReshapeOp> {
@@ -139,13 +95,6 @@ struct ExpandOpsPass : public memref::impl::ExpandOpsPassBase<ExpandOpsPass> {
     ConversionTarget target(ctx);
 
     target.addLegalDialect<arith::ArithDialect, memref::MemRefDialect>();
-    target.addDynamicallyLegalOp<memref::AtomicRMWOp>(
-        [](memref::AtomicRMWOp op) {
-          constexpr std::array shouldBeExpandedKinds = {
-              arith::AtomicRMWKind::maximumf, arith::AtomicRMWKind::minimumf,
-              arith::AtomicRMWKind::minnumf, arith::AtomicRMWKind::maxnumf};
-          return !llvm::is_contained(shouldBeExpandedKinds, op.getKind());
-        });
     target.addDynamicallyLegalOp<memref::ReshapeOp>([](memref::ReshapeOp op) {
       return !cast<MemRefType>(op.getShape().getType()).hasStaticShape();
     });
@@ -158,6 +107,5 @@ struct ExpandOpsPass : public memref::impl::ExpandOpsPassBase<ExpandOpsPass> {
 } // namespace
 
 void mlir::memref::populateExpandOpsPatterns(RewritePatternSet &patterns) {
-  patterns.add<AtomicRMWOpConverter, MemRefReshapeOpConverter>(
-      patterns.getContext());
+  patterns.add<MemRefReshapeOpConverter>(patterns.getContext());
 }
diff --git a/mlir/test/Dialect/MemRef/expand-ops.mlir b/mlir/test/Dialect/MemRef/expand-ops.mlir
index 65932b5814a6..fc8db546d918 100644
--- a/mlir/test/Dialect/MemRef/expand-ops.mlir
+++ b/mlir/test/Dialect/MemRef/expand-ops.mlir
@@ -1,42 +1,10 @@
 // RUN: mlir-opt -memref-expand %s -split-input-file | FileCheck %s
 
-// CHECK-LABEL: func @atomic_rmw_to_generic
-// CHECK-SAME: ([[F:%.*]]: memref<10xf32>, [[f:%.*]]: f32, [[i:%.*]]: index)
-func.func @atomic_rmw_to_generic(%F: memref<10xf32>, %f: f32, %i: index) -> f32 {
-  %a = memref.atomic_rmw maximumf %f, %F[%i] : (f32, memref<10xf32>) -> f32
-  %b = memref.atomic_rmw minimumf %f, %F[%i] : (f32, memref<10xf32>) -> f32
-  %c = memref.atomic_rmw maxnumf %f, %F[%i] : (f32, memref<10xf32>) -> f32
-  %d = memref.atomic_rmw minnumf %f, %F[%i] : (f32, memref<10xf32>) -> f32
-  return %a : f32
-}
-// CHECK: [[RESULT:%.*]] = memref.generic_atomic_rmw %arg0[%arg2] : memref<10xf32> {
-// CHECK: ^bb0([[CUR_VAL:%.*]]: f32):
-// CHECK:   [[MAXIMUM:%.*]] = arith.maximumf [[CUR_VAL]], [[f]] : f32
-// CHECK:   memref.atomic_yield [[MAXIMUM]] : f32
-// CHECK: }
-// CHECK: memref.generic_atomic_rmw %arg0[%arg2] : memref<10xf32> {
-// CHECK: ^bb0([[CUR_VAL:%.*]]: f32):
-// CHECK:   [[MINIMUM:%.*]] = arith.minimumf [[CUR_VAL]], [[f]] : f32
-// CHECK:   memref.atomic_yield [[MINIMUM]] : f32
-// CHECK: }
-// CHECK: memref.generic_atomic_rmw %arg0[%arg2] : memref<10xf32> {
-// CHECK: ^bb0([[CUR_VAL:%.*]]: f32):
-// CHECK:   [[MAXNUM:%.*]] = arith.maxnumf [[CUR_VAL]], [[f]] : f32
-// CHECK:   memref.atomic_yield [[MAXNUM]] : f32
-// CHECK: }
-// CHECK: memref.generic_atomic_rmw %arg0[%arg2] : memref<10xf32> {
-// CHECK: ^bb0([[CUR_VAL:%.*]]: f32):
-// CHECK:   [[MINNUM:%.*]] = arith.minnumf [[CUR_VAL]], [[f]] : f32
-// CHECK:   memref.atomic_yield [[MINNUM]] : f32
-// CHECK: }
-// CHECK: return [[RESULT]] : f32
-
-// -----
-
 // CHECK-LABEL: func @atomic_rmw_no_conversion
-func.func @atomic_rmw_no_conversion(%F: memref<10xf32>, %f: f32, %i: index) -> f32 {
+func.func @atomic_rmw_no_conversion(%F: memref<10xf32>, %f: f32, %i: index) -> (f32, f32) {
   %x = memref.atomic_rmw addf %f, %F[%i] : (f32, memref<10xf32>) -> f32
-  return %x : f32
+  %y = memref.atomic_rmw maximumf %f, %F[%i] : (f32, memref<10xf32>) -> f32
+  return %x, %y : f32, f32
 }
 // CHECK-NOT: generic_atomic_rmw
 
-- 
cgit v1.2.3


From d8e8ab79773f739c602c5869f80c6c5b5962c558 Mon Sep 17 00:00:00 2001
From: Benjamin Maxwell <benjamin.maxwell@arm.com>
Date: Wed, 18 Jun 2025 12:58:17 +0100
Subject: [AArch64][SME] Fix restoring callee-saves from FP with hazard padding
 (#143371)

Currently, when hazard-padding is enabled a (fixed-size) hazard slot is
placed in the CS area, just after the frame record. The size of this
slot is part of the "CalleeSaveBaseToFrameRecordOffset". The SVE
epilogue emission code assumed this offset was always zero, and
incorrectly setting the stack pointer, resulting in all SVE registers
being reloaded from incorrect offsets.

```
| prev_lr                           |
| prev_fp                           |
| (a.k.a. "frame record")           |
|-----------------------------------| <- fp(=x29)
|   <hazard padding>                |
|-----------------------------------| <- callee-saved base
|                                   |
| callee-saved fp/simd/SVE regs     |
|                                   |
|-----------------------------------| <- SVE callee-save base
```

i.e. in the above diagram, the code assumed `fp == callee-saved base`.
---
 llvm/lib/Target/AArch64/AArch64FrameLowering.cpp |   37 +-
 llvm/test/CodeGen/AArch64/stack-hazard.ll        | 1173 ++++++++++++++++++++++
 2 files changed, 1198 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 7ffe779f2408..a71668e71c23 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -2578,20 +2578,33 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF,
                     DeallocateAfter, TII, MachineInstr::FrameDestroy, false,
                     NeedsWinCFI, &HasWinCFI);
   } else if (SVEStackSize) {
-    // If we have stack realignment or variable sized objects on the stack,
-    // restore the stack pointer from the frame pointer prior to SVE CSR
-    // restoration.
-    if (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) {
-      if (int64_t CalleeSavedSize = AFI->getSVECalleeSavedStackSize()) {
-        // Set SP to start of SVE callee-save area from which they can
-        // be reloaded. The code below will deallocate the stack space
-        // space by moving FP -> SP.
-        emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, AArch64::FP,
-                        StackOffset::getScalable(-CalleeSavedSize), TII,
+    int64_t SVECalleeSavedSize = AFI->getSVECalleeSavedStackSize();
+    // If we have stack realignment or variable-sized objects we must use the
+    // FP to restore SVE callee saves (as there is an unknown amount of
+    // data/padding between the SP and SVE CS area).
+    Register BaseForSVEDealloc =
+        (AFI->isStackRealigned() || MFI.hasVarSizedObjects()) ? AArch64::FP
+                                                              : AArch64::SP;
+    if (SVECalleeSavedSize && BaseForSVEDealloc == AArch64::FP) {
+      Register CalleeSaveBase = AArch64::FP;
+      if (int64_t CalleeSaveBaseOffset =
+              AFI->getCalleeSaveBaseToFrameRecordOffset()) {
+        // If we have have an non-zero offset to the non-SVE CS base we need to
+        // compute the base address by subtracting the offest in a temporary
+        // register first (to avoid briefly deallocating the SVE CS).
+        CalleeSaveBase = MBB.getParent()->getRegInfo().createVirtualRegister(
+            &AArch64::GPR64RegClass);
+        emitFrameOffset(MBB, RestoreBegin, DL, CalleeSaveBase, AArch64::FP,
+                        StackOffset::getFixed(-CalleeSaveBaseOffset), TII,
                         MachineInstr::FrameDestroy);
       }
-    } else {
-      if (AFI->getSVECalleeSavedStackSize()) {
+      // The code below will deallocate the stack space space by moving the
+      // SP to the start of the SVE callee-save area.
+      emitFrameOffset(MBB, RestoreBegin, DL, AArch64::SP, CalleeSaveBase,
+                      StackOffset::getScalable(-SVECalleeSavedSize), TII,
+                      MachineInstr::FrameDestroy);
+    } else if (BaseForSVEDealloc == AArch64::SP) {
+      if (SVECalleeSavedSize) {
         // Deallocate the non-SVE locals first before we can deallocate (and
         // restore callee saves) from the SVE area.
         emitFrameOffset(
diff --git a/llvm/test/CodeGen/AArch64/stack-hazard.ll b/llvm/test/CodeGen/AArch64/stack-hazard.ll
index e169b199733b..3a3340520013 100644
--- a/llvm/test/CodeGen/AArch64/stack-hazard.ll
+++ b/llvm/test/CodeGen/AArch64/stack-hazard.ll
@@ -3143,3 +3143,1176 @@ entry:
   call void @bar(ptr noundef nonnull %b)
   ret i32 0
 }
+
+
+define i32 @svecc_call_dynamic_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
+; CHECK0-LABEL: svecc_call_dynamic_alloca:
+; CHECK0:       // %bb.0: // %entry
+; CHECK0-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_def_cfa_offset 64
+; CHECK0-NEXT:    cntd x9
+; CHECK0-NEXT:    stp x27, x26, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x9, x28, [sp, #16] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 64
+; CHECK0-NEXT:    .cfi_offset w19, -8
+; CHECK0-NEXT:    .cfi_offset w20, -16
+; CHECK0-NEXT:    .cfi_offset w26, -24
+; CHECK0-NEXT:    .cfi_offset w27, -32
+; CHECK0-NEXT:    .cfi_offset w28, -40
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
+; CHECK0-NEXT:    addvl sp, sp, #-18
+; CHECK0-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 64 - 8 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 64 - 16 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 64 - 24 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 64 - 32 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 64 - 40 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 64 - 48 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 64 - 56 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 64 - 64 * VG
+; CHECK0-NEXT:    mov w9, w0
+; CHECK0-NEXT:    mov x8, sp
+; CHECK0-NEXT:    mov w2, w1
+; CHECK0-NEXT:    add x9, x9, #15
+; CHECK0-NEXT:    mov x19, sp
+; CHECK0-NEXT:    and x9, x9, #0x1fffffff0
+; CHECK0-NEXT:    sub x8, x8, x9
+; CHECK0-NEXT:    mov sp, x8
+; CHECK0-NEXT:    //APP
+; CHECK0-NEXT:    //NO_APP
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    and x20, x0, #0x1
+; CHECK0-NEXT:    .cfi_offset vg, -48
+; CHECK0-NEXT:    tbz w20, #0, .LBB35_2
+; CHECK0-NEXT:  // %bb.1: // %entry
+; CHECK0-NEXT:    smstop sm
+; CHECK0-NEXT:  .LBB35_2: // %entry
+; CHECK0-NEXT:    mov x0, x8
+; CHECK0-NEXT:    mov w1, #45 // =0x2d
+; CHECK0-NEXT:    bl memset
+; CHECK0-NEXT:    tbz w20, #0, .LBB35_4
+; CHECK0-NEXT:  // %bb.3: // %entry
+; CHECK0-NEXT:    smstart sm
+; CHECK0-NEXT:  .LBB35_4: // %entry
+; CHECK0-NEXT:    mov w0, #22647 // =0x5877
+; CHECK0-NEXT:    movk w0, #59491, lsl #16
+; CHECK0-NEXT:    .cfi_restore vg
+; CHECK0-NEXT:    addvl sp, x29, #-18
+; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    .cfi_restore z8
+; CHECK0-NEXT:    .cfi_restore z9
+; CHECK0-NEXT:    .cfi_restore z10
+; CHECK0-NEXT:    .cfi_restore z11
+; CHECK0-NEXT:    .cfi_restore z12
+; CHECK0-NEXT:    .cfi_restore z13
+; CHECK0-NEXT:    .cfi_restore z14
+; CHECK0-NEXT:    .cfi_restore z15
+; CHECK0-NEXT:    mov sp, x29
+; CHECK0-NEXT:    .cfi_def_cfa wsp, 64
+; CHECK0-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr x28, [sp, #24] // 8-byte Folded Reload
+; CHECK0-NEXT:    ldp x27, x26, [sp, #32] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
+; CHECK0-NEXT:    .cfi_def_cfa_offset 0
+; CHECK0-NEXT:    .cfi_restore w19
+; CHECK0-NEXT:    .cfi_restore w20
+; CHECK0-NEXT:    .cfi_restore w26
+; CHECK0-NEXT:    .cfi_restore w27
+; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore w30
+; CHECK0-NEXT:    .cfi_restore w29
+; CHECK0-NEXT:    ret
+;
+; CHECK64-LABEL: svecc_call_dynamic_alloca:
+; CHECK64:       // %bb.0: // %entry
+; CHECK64-NEXT:    sub sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 128
+; CHECK64-NEXT:    cntd x9
+; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x9, x28, [sp, #80] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x27, x26, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x20, x19, [sp, #112] // 16-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    .cfi_def_cfa w29, 64
+; CHECK64-NEXT:    .cfi_offset w19, -8
+; CHECK64-NEXT:    .cfi_offset w20, -16
+; CHECK64-NEXT:    .cfi_offset w26, -24
+; CHECK64-NEXT:    .cfi_offset w27, -32
+; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset w30, -56
+; CHECK64-NEXT:    .cfi_offset w29, -64
+; CHECK64-NEXT:    addvl sp, sp, #-18
+; CHECK64-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 128 - 8 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 128 - 16 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 128 - 24 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 128 - 32 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 128 - 40 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 128 - 48 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 128 - 56 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 128 - 64 * VG
+; CHECK64-NEXT:    sub sp, sp, #64
+; CHECK64-NEXT:    mov w9, w0
+; CHECK64-NEXT:    mov x8, sp
+; CHECK64-NEXT:    mov w2, w1
+; CHECK64-NEXT:    add x9, x9, #15
+; CHECK64-NEXT:    mov x19, sp
+; CHECK64-NEXT:    and x9, x9, #0x1fffffff0
+; CHECK64-NEXT:    sub x8, x8, x9
+; CHECK64-NEXT:    mov sp, x8
+; CHECK64-NEXT:    //APP
+; CHECK64-NEXT:    //NO_APP
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    and x20, x0, #0x1
+; CHECK64-NEXT:    .cfi_offset vg, -48
+; CHECK64-NEXT:    tbz w20, #0, .LBB35_2
+; CHECK64-NEXT:  // %bb.1: // %entry
+; CHECK64-NEXT:    smstop sm
+; CHECK64-NEXT:  .LBB35_2: // %entry
+; CHECK64-NEXT:    mov x0, x8
+; CHECK64-NEXT:    mov w1, #45 // =0x2d
+; CHECK64-NEXT:    bl memset
+; CHECK64-NEXT:    tbz w20, #0, .LBB35_4
+; CHECK64-NEXT:  // %bb.3: // %entry
+; CHECK64-NEXT:    smstart sm
+; CHECK64-NEXT:  .LBB35_4: // %entry
+; CHECK64-NEXT:    mov w0, #22647 // =0x5877
+; CHECK64-NEXT:    movk w0, #59491, lsl #16
+; CHECK64-NEXT:    .cfi_restore vg
+; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    addvl sp, x8, #-18
+; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    .cfi_restore z8
+; CHECK64-NEXT:    .cfi_restore z9
+; CHECK64-NEXT:    .cfi_restore z10
+; CHECK64-NEXT:    .cfi_restore z11
+; CHECK64-NEXT:    .cfi_restore z12
+; CHECK64-NEXT:    .cfi_restore z13
+; CHECK64-NEXT:    .cfi_restore z14
+; CHECK64-NEXT:    .cfi_restore z15
+; CHECK64-NEXT:    sub sp, x29, #64
+; CHECK64-NEXT:    .cfi_def_cfa wsp, 128
+; CHECK64-NEXT:    ldp x20, x19, [sp, #112] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr x28, [sp, #88] // 8-byte Folded Reload
+; CHECK64-NEXT:    ldp x27, x26, [sp, #96] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK64-NEXT:    add sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 0
+; CHECK64-NEXT:    .cfi_restore w19
+; CHECK64-NEXT:    .cfi_restore w20
+; CHECK64-NEXT:    .cfi_restore w26
+; CHECK64-NEXT:    .cfi_restore w27
+; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore w30
+; CHECK64-NEXT:    .cfi_restore w29
+; CHECK64-NEXT:    ret
+;
+; CHECK1024-LABEL: svecc_call_dynamic_alloca:
+; CHECK1024:       // %bb.0: // %entry
+; CHECK1024-NEXT:    sub sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 1088
+; CHECK1024-NEXT:    cntd x9
+; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x9, [sp, #1040] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x28, [sp, #1048] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x27, [sp, #1056] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x26, [sp, #1064] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x20, [sp, #1072] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1080] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 64
+; CHECK1024-NEXT:    .cfi_offset w19, -8
+; CHECK1024-NEXT:    .cfi_offset w20, -16
+; CHECK1024-NEXT:    .cfi_offset w26, -24
+; CHECK1024-NEXT:    .cfi_offset w27, -32
+; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset w30, -56
+; CHECK1024-NEXT:    .cfi_offset w29, -64
+; CHECK1024-NEXT:    addvl sp, sp, #-18
+; CHECK1024-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 1088 - 8 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 1088 - 16 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 1088 - 24 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 1088 - 32 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 1088 - 40 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 1088 - 48 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 1088 - 56 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 1088 - 64 * VG
+; CHECK1024-NEXT:    sub sp, sp, #1024
+; CHECK1024-NEXT:    mov w9, w0
+; CHECK1024-NEXT:    mov x8, sp
+; CHECK1024-NEXT:    mov w2, w1
+; CHECK1024-NEXT:    add x9, x9, #15
+; CHECK1024-NEXT:    mov x19, sp
+; CHECK1024-NEXT:    and x9, x9, #0x1fffffff0
+; CHECK1024-NEXT:    sub x8, x8, x9
+; CHECK1024-NEXT:    mov sp, x8
+; CHECK1024-NEXT:    //APP
+; CHECK1024-NEXT:    //NO_APP
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    and x20, x0, #0x1
+; CHECK1024-NEXT:    .cfi_offset vg, -48
+; CHECK1024-NEXT:    tbz w20, #0, .LBB35_2
+; CHECK1024-NEXT:  // %bb.1: // %entry
+; CHECK1024-NEXT:    smstop sm
+; CHECK1024-NEXT:  .LBB35_2: // %entry
+; CHECK1024-NEXT:    mov x0, x8
+; CHECK1024-NEXT:    mov w1, #45 // =0x2d
+; CHECK1024-NEXT:    bl memset
+; CHECK1024-NEXT:    tbz w20, #0, .LBB35_4
+; CHECK1024-NEXT:  // %bb.3: // %entry
+; CHECK1024-NEXT:    smstart sm
+; CHECK1024-NEXT:  .LBB35_4: // %entry
+; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
+; CHECK1024-NEXT:    movk w0, #59491, lsl #16
+; CHECK1024-NEXT:    .cfi_restore vg
+; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    addvl sp, x8, #-18
+; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    .cfi_restore z8
+; CHECK1024-NEXT:    .cfi_restore z9
+; CHECK1024-NEXT:    .cfi_restore z10
+; CHECK1024-NEXT:    .cfi_restore z11
+; CHECK1024-NEXT:    .cfi_restore z12
+; CHECK1024-NEXT:    .cfi_restore z13
+; CHECK1024-NEXT:    .cfi_restore z14
+; CHECK1024-NEXT:    .cfi_restore z15
+; CHECK1024-NEXT:    sub sp, x29, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1088
+; CHECK1024-NEXT:    ldr x19, [sp, #1080] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x20, [sp, #1072] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x26, [sp, #1064] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x27, [sp, #1056] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x28, [sp, #1048] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
+; CHECK1024-NEXT:    add sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 0
+; CHECK1024-NEXT:    .cfi_restore w19
+; CHECK1024-NEXT:    .cfi_restore w20
+; CHECK1024-NEXT:    .cfi_restore w26
+; CHECK1024-NEXT:    .cfi_restore w27
+; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore w30
+; CHECK1024-NEXT:    .cfi_restore w29
+; CHECK1024-NEXT:    ret
+entry:
+  %ptr = alloca i8, i32 %P1
+  tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
+  %call = call ptr @memset(ptr noundef nonnull %ptr, i32 noundef 45, i32 noundef %P2)
+  ret i32 -396142473
+}
+
+
+define i32 @svecc_call_realign(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
+; CHECK0-LABEL: svecc_call_realign:
+; CHECK0:       // %bb.0: // %entry
+; CHECK0-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_def_cfa_offset 64
+; CHECK0-NEXT:    cntd x9
+; CHECK0-NEXT:    stp x28, x27, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    str x9, [sp, #16] // 8-byte Folded Spill
+; CHECK0-NEXT:    stp x26, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 64
+; CHECK0-NEXT:    .cfi_offset w19, -8
+; CHECK0-NEXT:    .cfi_offset w26, -16
+; CHECK0-NEXT:    .cfi_offset w27, -24
+; CHECK0-NEXT:    .cfi_offset w28, -32
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
+; CHECK0-NEXT:    addvl sp, sp, #-18
+; CHECK0-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 64 - 8 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 64 - 16 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 64 - 24 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 64 - 32 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 64 - 40 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 64 - 48 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 64 - 56 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 64 - 64 * VG
+; CHECK0-NEXT:    sub x9, sp, #1024
+; CHECK0-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK0-NEXT:    mov w2, w1
+; CHECK0-NEXT:    //APP
+; CHECK0-NEXT:    //NO_APP
+; CHECK0-NEXT:    bl __arm_sme_state
+; CHECK0-NEXT:    and x19, x0, #0x1
+; CHECK0-NEXT:    .cfi_offset vg, -48
+; CHECK0-NEXT:    tbz w19, #0, .LBB36_2
+; CHECK0-NEXT:  // %bb.1: // %entry
+; CHECK0-NEXT:    smstop sm
+; CHECK0-NEXT:  .LBB36_2: // %entry
+; CHECK0-NEXT:    mov x0, sp
+; CHECK0-NEXT:    mov w1, #45 // =0x2d
+; CHECK0-NEXT:    bl memset
+; CHECK0-NEXT:    tbz w19, #0, .LBB36_4
+; CHECK0-NEXT:  // %bb.3: // %entry
+; CHECK0-NEXT:    smstart sm
+; CHECK0-NEXT:  .LBB36_4: // %entry
+; CHECK0-NEXT:    mov w0, #22647 // =0x5877
+; CHECK0-NEXT:    movk w0, #59491, lsl #16
+; CHECK0-NEXT:    .cfi_restore vg
+; CHECK0-NEXT:    addvl sp, x29, #-18
+; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    .cfi_restore z8
+; CHECK0-NEXT:    .cfi_restore z9
+; CHECK0-NEXT:    .cfi_restore z10
+; CHECK0-NEXT:    .cfi_restore z11
+; CHECK0-NEXT:    .cfi_restore z12
+; CHECK0-NEXT:    .cfi_restore z13
+; CHECK0-NEXT:    .cfi_restore z14
+; CHECK0-NEXT:    .cfi_restore z15
+; CHECK0-NEXT:    mov sp, x29
+; CHECK0-NEXT:    .cfi_def_cfa wsp, 64
+; CHECK0-NEXT:    ldp x26, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x28, x27, [sp, #32] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
+; CHECK0-NEXT:    .cfi_def_cfa_offset 0
+; CHECK0-NEXT:    .cfi_restore w19
+; CHECK0-NEXT:    .cfi_restore w26
+; CHECK0-NEXT:    .cfi_restore w27
+; CHECK0-NEXT:    .cfi_restore w28
+; CHECK0-NEXT:    .cfi_restore w30
+; CHECK0-NEXT:    .cfi_restore w29
+; CHECK0-NEXT:    ret
+;
+; CHECK64-LABEL: svecc_call_realign:
+; CHECK64:       // %bb.0: // %entry
+; CHECK64-NEXT:    sub sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 128
+; CHECK64-NEXT:    cntd x9
+; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x9, x28, [sp, #80] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x27, x26, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    str x19, [sp, #112] // 8-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    .cfi_def_cfa w29, 64
+; CHECK64-NEXT:    .cfi_offset w19, -16
+; CHECK64-NEXT:    .cfi_offset w26, -24
+; CHECK64-NEXT:    .cfi_offset w27, -32
+; CHECK64-NEXT:    .cfi_offset w28, -40
+; CHECK64-NEXT:    .cfi_offset w30, -56
+; CHECK64-NEXT:    .cfi_offset w29, -64
+; CHECK64-NEXT:    addvl sp, sp, #-18
+; CHECK64-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 128 - 8 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 128 - 16 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 128 - 24 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 128 - 32 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 128 - 40 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 128 - 48 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 128 - 56 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 128 - 64 * VG
+; CHECK64-NEXT:    sub x9, sp, #1088
+; CHECK64-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK64-NEXT:    mov w2, w1
+; CHECK64-NEXT:    //APP
+; CHECK64-NEXT:    //NO_APP
+; CHECK64-NEXT:    bl __arm_sme_state
+; CHECK64-NEXT:    and x19, x0, #0x1
+; CHECK64-NEXT:    .cfi_offset vg, -48
+; CHECK64-NEXT:    tbz w19, #0, .LBB36_2
+; CHECK64-NEXT:  // %bb.1: // %entry
+; CHECK64-NEXT:    smstop sm
+; CHECK64-NEXT:  .LBB36_2: // %entry
+; CHECK64-NEXT:    mov x0, sp
+; CHECK64-NEXT:    mov w1, #45 // =0x2d
+; CHECK64-NEXT:    bl memset
+; CHECK64-NEXT:    tbz w19, #0, .LBB36_4
+; CHECK64-NEXT:  // %bb.3: // %entry
+; CHECK64-NEXT:    smstart sm
+; CHECK64-NEXT:  .LBB36_4: // %entry
+; CHECK64-NEXT:    mov w0, #22647 // =0x5877
+; CHECK64-NEXT:    movk w0, #59491, lsl #16
+; CHECK64-NEXT:    .cfi_restore vg
+; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    addvl sp, x8, #-18
+; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    .cfi_restore z8
+; CHECK64-NEXT:    .cfi_restore z9
+; CHECK64-NEXT:    .cfi_restore z10
+; CHECK64-NEXT:    .cfi_restore z11
+; CHECK64-NEXT:    .cfi_restore z12
+; CHECK64-NEXT:    .cfi_restore z13
+; CHECK64-NEXT:    .cfi_restore z14
+; CHECK64-NEXT:    .cfi_restore z15
+; CHECK64-NEXT:    sub sp, x29, #64
+; CHECK64-NEXT:    .cfi_def_cfa wsp, 128
+; CHECK64-NEXT:    ldp x26, x19, [sp, #104] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x28, x27, [sp, #88] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x29, x30, [sp, #64] // 16-byte Folded Reload
+; CHECK64-NEXT:    add sp, sp, #128
+; CHECK64-NEXT:    .cfi_def_cfa_offset 0
+; CHECK64-NEXT:    .cfi_restore w19
+; CHECK64-NEXT:    .cfi_restore w26
+; CHECK64-NEXT:    .cfi_restore w27
+; CHECK64-NEXT:    .cfi_restore w28
+; CHECK64-NEXT:    .cfi_restore w30
+; CHECK64-NEXT:    .cfi_restore w29
+; CHECK64-NEXT:    ret
+;
+; CHECK1024-LABEL: svecc_call_realign:
+; CHECK1024:       // %bb.0: // %entry
+; CHECK1024-NEXT:    sub sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 1088
+; CHECK1024-NEXT:    cntd x9
+; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x9, [sp, #1040] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x28, [sp, #1048] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x27, [sp, #1056] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x26, [sp, #1064] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1072] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 64
+; CHECK1024-NEXT:    .cfi_offset w19, -16
+; CHECK1024-NEXT:    .cfi_offset w26, -24
+; CHECK1024-NEXT:    .cfi_offset w27, -32
+; CHECK1024-NEXT:    .cfi_offset w28, -40
+; CHECK1024-NEXT:    .cfi_offset w30, -56
+; CHECK1024-NEXT:    .cfi_offset w29, -64
+; CHECK1024-NEXT:    addvl sp, sp, #-18
+; CHECK1024-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 1088 - 8 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 1088 - 16 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 1088 - 24 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 1088 - 32 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 1088 - 40 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 1088 - 48 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 1088 - 56 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 1088 - 64 * VG
+; CHECK1024-NEXT:    sub x9, sp, #2048
+; CHECK1024-NEXT:    and sp, x9, #0xffffffffffffffe0
+; CHECK1024-NEXT:    mov w2, w1
+; CHECK1024-NEXT:    //APP
+; CHECK1024-NEXT:    //NO_APP
+; CHECK1024-NEXT:    bl __arm_sme_state
+; CHECK1024-NEXT:    and x19, x0, #0x1
+; CHECK1024-NEXT:    .cfi_offset vg, -48
+; CHECK1024-NEXT:    tbz w19, #0, .LBB36_2
+; CHECK1024-NEXT:  // %bb.1: // %entry
+; CHECK1024-NEXT:    smstop sm
+; CHECK1024-NEXT:  .LBB36_2: // %entry
+; CHECK1024-NEXT:    mov x0, sp
+; CHECK1024-NEXT:    mov w1, #45 // =0x2d
+; CHECK1024-NEXT:    bl memset
+; CHECK1024-NEXT:    tbz w19, #0, .LBB36_4
+; CHECK1024-NEXT:  // %bb.3: // %entry
+; CHECK1024-NEXT:    smstart sm
+; CHECK1024-NEXT:  .LBB36_4: // %entry
+; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
+; CHECK1024-NEXT:    movk w0, #59491, lsl #16
+; CHECK1024-NEXT:    .cfi_restore vg
+; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    addvl sp, x8, #-18
+; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    .cfi_restore z8
+; CHECK1024-NEXT:    .cfi_restore z9
+; CHECK1024-NEXT:    .cfi_restore z10
+; CHECK1024-NEXT:    .cfi_restore z11
+; CHECK1024-NEXT:    .cfi_restore z12
+; CHECK1024-NEXT:    .cfi_restore z13
+; CHECK1024-NEXT:    .cfi_restore z14
+; CHECK1024-NEXT:    .cfi_restore z15
+; CHECK1024-NEXT:    sub sp, x29, #1024
+; CHECK1024-NEXT:    .cfi_def_cfa wsp, 1088
+; CHECK1024-NEXT:    ldr x19, [sp, #1072] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x26, [sp, #1064] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x27, [sp, #1056] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x28, [sp, #1048] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
+; CHECK1024-NEXT:    add sp, sp, #1088
+; CHECK1024-NEXT:    .cfi_def_cfa_offset 0
+; CHECK1024-NEXT:    .cfi_restore w19
+; CHECK1024-NEXT:    .cfi_restore w26
+; CHECK1024-NEXT:    .cfi_restore w27
+; CHECK1024-NEXT:    .cfi_restore w28
+; CHECK1024-NEXT:    .cfi_restore w30
+; CHECK1024-NEXT:    .cfi_restore w29
+; CHECK1024-NEXT:    ret
+entry:
+  %ptr = alloca i8, i32 1000, align 32
+  tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
+  %call = call ptr @memset(ptr noundef nonnull %ptr, i32 noundef 45, i32 noundef %P2)
+  ret i32 -396142473
+}
+
+
+define i32 @svecc_call_dynamic_and_scalable_alloca(<4 x i16> %P0, i32 %P1, i32 %P2, <vscale x 16 x i8> %P3, i16 %P4) "aarch64_pstate_sm_compatible" {
+; CHECK0-LABEL: svecc_call_dynamic_and_scalable_alloca:
+; CHECK0:       // %bb.0: // %entry
+; CHECK0-NEXT:    stp x29, x30, [sp, #-64]! // 16-byte Folded Spill
+; CHECK0-NEXT:    str x28, [sp, #16] // 8-byte Folded Spill
+; CHECK0-NEXT:    mov x29, sp
+; CHECK0-NEXT:    stp x27, x26, [sp, #32] // 16-byte Folded Spill
+; CHECK0-NEXT:    stp x20, x19, [sp, #48] // 16-byte Folded Spill
+; CHECK0-NEXT:    addvl sp, sp, #-18
+; CHECK0-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK0-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK0-NEXT:    sub sp, sp, #48
+; CHECK0-NEXT:    addvl sp, sp, #-1
+; CHECK0-NEXT:    mov x19, sp
+; CHECK0-NEXT:    .cfi_def_cfa w29, 64
+; CHECK0-NEXT:    .cfi_offset w19, -8
+; CHECK0-NEXT:    .cfi_offset w20, -16
+; CHECK0-NEXT:    .cfi_offset w26, -24
+; CHECK0-NEXT:    .cfi_offset w27, -32
+; CHECK0-NEXT:    .cfi_offset w28, -48
+; CHECK0-NEXT:    .cfi_offset w30, -56
+; CHECK0-NEXT:    .cfi_offset w29, -64
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x48, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 64 - 8 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x49, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 64 - 16 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4a, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 64 - 24 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4b, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 64 - 32 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4c, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 64 - 40 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4d, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 64 - 48 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4e, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 64 - 56 * VG
+; CHECK0-NEXT:    .cfi_escape 0x10, 0x4f, 0x0a, 0x11, 0x40, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 64 - 64 * VG
+; CHECK0-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK0-NEXT:    ubfiz x8, x0, #2, #32
+; CHECK0-NEXT:    mov x9, sp
+; CHECK0-NEXT:    add x8, x8, #15
+; CHECK0-NEXT:    and x8, x8, #0x7fffffff0
+; CHECK0-NEXT:    sub x20, x9, x8
+; CHECK0-NEXT:    mov sp, x20
+; CHECK0-NEXT:    //APP
+; CHECK0-NEXT:    //NO_APP
+; CHECK0-NEXT:    add x0, x19, #8
+; CHECK0-NEXT:    bl bar
+; CHECK0-NEXT:    addvl x0, x29, #-19
+; CHECK0-NEXT:    bl bar
+; CHECK0-NEXT:    mov x0, x20
+; CHECK0-NEXT:    bl bar
+; CHECK0-NEXT:    mov w0, #22647 // =0x5877
+; CHECK0-NEXT:    movk w0, #59491, lsl #16
+; CHECK0-NEXT:    addvl sp, x29, #-18
+; CHECK0-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK0-NEXT:    mov sp, x29
+; CHECK0-NEXT:    ldp x20, x19, [sp, #48] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldr x28, [sp, #16] // 8-byte Folded Reload
+; CHECK0-NEXT:    ldp x27, x26, [sp, #32] // 16-byte Folded Reload
+; CHECK0-NEXT:    ldp x29, x30, [sp], #64 // 16-byte Folded Reload
+; CHECK0-NEXT:    ret
+;
+; CHECK64-LABEL: svecc_call_dynamic_and_scalable_alloca:
+; CHECK64:       // %bb.0: // %entry
+; CHECK64-NEXT:    sub sp, sp, #128
+; CHECK64-NEXT:    stp x29, x30, [sp, #64] // 16-byte Folded Spill
+; CHECK64-NEXT:    add x29, sp, #64
+; CHECK64-NEXT:    stp x28, x27, [sp, #80] // 16-byte Folded Spill
+; CHECK64-NEXT:    stp x26, x20, [sp, #96] // 16-byte Folded Spill
+; CHECK64-NEXT:    str x19, [sp, #112] // 8-byte Folded Spill
+; CHECK64-NEXT:    addvl sp, sp, #-18
+; CHECK64-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK64-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK64-NEXT:    sub sp, sp, #112
+; CHECK64-NEXT:    addvl sp, sp, #-1
+; CHECK64-NEXT:    mov x19, sp
+; CHECK64-NEXT:    .cfi_def_cfa w29, 64
+; CHECK64-NEXT:    .cfi_offset w19, -16
+; CHECK64-NEXT:    .cfi_offset w20, -24
+; CHECK64-NEXT:    .cfi_offset w26, -32
+; CHECK64-NEXT:    .cfi_offset w27, -40
+; CHECK64-NEXT:    .cfi_offset w28, -48
+; CHECK64-NEXT:    .cfi_offset w30, -56
+; CHECK64-NEXT:    .cfi_offset w29, -64
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 128 - 8 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 128 - 16 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 128 - 24 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 128 - 32 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 128 - 40 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 128 - 48 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 128 - 56 * VG
+; CHECK64-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0x80, 0x7f, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 128 - 64 * VG
+; CHECK64-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK64-NEXT:    ubfiz x8, x0, #2, #32
+; CHECK64-NEXT:    mov x9, sp
+; CHECK64-NEXT:    add x8, x8, #15
+; CHECK64-NEXT:    and x8, x8, #0x7fffffff0
+; CHECK64-NEXT:    sub x20, x9, x8
+; CHECK64-NEXT:    mov sp, x20
+; CHECK64-NEXT:    //APP
+; CHECK64-NEXT:    //NO_APP
+; CHECK64-NEXT:    add x0, x19, #8
+; CHECK64-NEXT:    bl bar
+; CHECK64-NEXT:    sub x0, x29, #64
+; CHECK64-NEXT:    addvl x0, x0, #-19
+; CHECK64-NEXT:    bl bar
+; CHECK64-NEXT:    mov x0, x20
+; CHECK64-NEXT:    bl bar
+; CHECK64-NEXT:    mov w0, #22647 // =0x5877
+; CHECK64-NEXT:    sub x8, x29, #64
+; CHECK64-NEXT:    movk w0, #59491, lsl #16
+; CHECK64-NEXT:    addvl sp, x8, #-18
+; CHECK64-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK64-NEXT:    sub sp, x29, #64
+; CHECK64-NEXT:    ldp x20, x19, [sp, #104] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldr x29, [sp, #64] // 8-byte Folded Reload
+; CHECK64-NEXT:    ldp x27, x26, [sp, #88] // 16-byte Folded Reload
+; CHECK64-NEXT:    ldp x30, x28, [sp, #72] // 16-byte Folded Reload
+; CHECK64-NEXT:    add sp, sp, #128
+; CHECK64-NEXT:    ret
+;
+; CHECK1024-LABEL: svecc_call_dynamic_and_scalable_alloca:
+; CHECK1024:       // %bb.0: // %entry
+; CHECK1024-NEXT:    sub sp, sp, #1088
+; CHECK1024-NEXT:    str x29, [sp, #1024] // 8-byte Folded Spill
+; CHECK1024-NEXT:    add x29, sp, #1024
+; CHECK1024-NEXT:    str x30, [sp, #1032] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x28, [sp, #1040] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x27, [sp, #1048] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x26, [sp, #1056] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x20, [sp, #1064] // 8-byte Folded Spill
+; CHECK1024-NEXT:    str x19, [sp, #1072] // 8-byte Folded Spill
+; CHECK1024-NEXT:    addvl sp, sp, #-18
+; CHECK1024-NEXT:    str p15, [sp, #4, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p14, [sp, #5, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p13, [sp, #6, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p12, [sp, #7, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p11, [sp, #8, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p10, [sp, #9, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p9, [sp, #10, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p8, [sp, #11, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p7, [sp, #12, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p6, [sp, #13, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p5, [sp, #14, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str p4, [sp, #15, mul vl] // 2-byte Folded Spill
+; CHECK1024-NEXT:    str z23, [sp, #2, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z22, [sp, #3, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z21, [sp, #4, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z20, [sp, #5, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z19, [sp, #6, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z18, [sp, #7, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z17, [sp, #8, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z16, [sp, #9, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z15, [sp, #10, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z14, [sp, #11, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z13, [sp, #12, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z12, [sp, #13, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z11, [sp, #14, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z10, [sp, #15, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z9, [sp, #16, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    str z8, [sp, #17, mul vl] // 16-byte Folded Spill
+; CHECK1024-NEXT:    sub sp, sp, #1072
+; CHECK1024-NEXT:    addvl sp, sp, #-1
+; CHECK1024-NEXT:    mov x19, sp
+; CHECK1024-NEXT:    .cfi_def_cfa w29, 64
+; CHECK1024-NEXT:    .cfi_offset w19, -16
+; CHECK1024-NEXT:    .cfi_offset w20, -24
+; CHECK1024-NEXT:    .cfi_offset w26, -32
+; CHECK1024-NEXT:    .cfi_offset w27, -40
+; CHECK1024-NEXT:    .cfi_offset w28, -48
+; CHECK1024-NEXT:    .cfi_offset w30, -56
+; CHECK1024-NEXT:    .cfi_offset w29, -64
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x48, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x78, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d8 @ cfa - 1088 - 8 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x49, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x70, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d9 @ cfa - 1088 - 16 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4a, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x68, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d10 @ cfa - 1088 - 24 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4b, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x60, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d11 @ cfa - 1088 - 32 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4c, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x58, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d12 @ cfa - 1088 - 40 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4d, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x50, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d13 @ cfa - 1088 - 48 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4e, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x48, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d14 @ cfa - 1088 - 56 * VG
+; CHECK1024-NEXT:    .cfi_escape 0x10, 0x4f, 0x0b, 0x11, 0xc0, 0x77, 0x22, 0x11, 0x40, 0x92, 0x2e, 0x00, 0x1e, 0x22 // $d15 @ cfa - 1088 - 64 * VG
+; CHECK1024-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK1024-NEXT:    ubfiz x8, x0, #2, #32
+; CHECK1024-NEXT:    mov x9, sp
+; CHECK1024-NEXT:    add x8, x8, #15
+; CHECK1024-NEXT:    and x8, x8, #0x7fffffff0
+; CHECK1024-NEXT:    sub x20, x9, x8
+; CHECK1024-NEXT:    mov sp, x20
+; CHECK1024-NEXT:    //APP
+; CHECK1024-NEXT:    //NO_APP
+; CHECK1024-NEXT:    add x0, x19, #8
+; CHECK1024-NEXT:    bl bar
+; CHECK1024-NEXT:    sub x0, x29, #1024
+; CHECK1024-NEXT:    addvl x0, x0, #-19
+; CHECK1024-NEXT:    bl bar
+; CHECK1024-NEXT:    mov x0, x20
+; CHECK1024-NEXT:    bl bar
+; CHECK1024-NEXT:    mov w0, #22647 // =0x5877
+; CHECK1024-NEXT:    sub x8, x29, #1024
+; CHECK1024-NEXT:    movk w0, #59491, lsl #16
+; CHECK1024-NEXT:    addvl sp, x8, #-18
+; CHECK1024-NEXT:    ldr z23, [sp, #2, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z22, [sp, #3, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z21, [sp, #4, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z20, [sp, #5, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z19, [sp, #6, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z18, [sp, #7, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z17, [sp, #8, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z16, [sp, #9, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z15, [sp, #10, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z14, [sp, #11, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z13, [sp, #12, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z12, [sp, #13, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z11, [sp, #14, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z10, [sp, #15, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z9, [sp, #16, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr z8, [sp, #17, mul vl] // 16-byte Folded Reload
+; CHECK1024-NEXT:    ldr p15, [sp, #4, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p14, [sp, #5, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p13, [sp, #6, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p12, [sp, #7, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p11, [sp, #8, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p10, [sp, #9, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p9, [sp, #10, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p8, [sp, #11, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p7, [sp, #12, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p6, [sp, #13, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p5, [sp, #14, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    ldr p4, [sp, #15, mul vl] // 2-byte Folded Reload
+; CHECK1024-NEXT:    sub sp, x29, #1024
+; CHECK1024-NEXT:    ldr x19, [sp, #1072] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x20, [sp, #1064] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x26, [sp, #1056] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x27, [sp, #1048] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x28, [sp, #1040] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x30, [sp, #1032] // 8-byte Folded Reload
+; CHECK1024-NEXT:    ldr x29, [sp, #1024] // 8-byte Folded Reload
+; CHECK1024-NEXT:    add sp, sp, #1088
+; CHECK1024-NEXT:    ret
+entry:
+  %a = alloca i32, i32 10
+  %b = alloca <vscale x 4 x i32>
+  %c = alloca i32, i32 %P1, align 4
+  tail call void asm sideeffect "", "~{x0},~{x28},~{x27},~{x3}"() #2
+  call void @bar(ptr noundef nonnull %a)
+  call void @bar(ptr noundef nonnull %b)
+  call void @bar(ptr noundef nonnull %c)
+  ret i32 -396142473
+}
-- 
cgit v1.2.3


From 34a48941498d95ec2682f7adaeb6115b7b4d70ba Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Wed, 18 Jun 2025 13:06:49 +0100
Subject: [X86] detectZextAbsDiff - use SDPatternMatch::m_Abs() matcher. NFC.

---
 llvm/lib/Target/X86/X86ISelLowering.cpp | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1ca5fc5376f0..4751361c71f2 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -46009,11 +46009,9 @@ static bool detectZextAbsDiff(SDValue Abs, SDValue &Op0, SDValue &Op1) {
 
   // Check if the operands of the sub are zero-extended from vectors of i8.
   EVT SrcVT0, SrcVT1;
-  return sd_match(
-             Abs,
-             m_UnaryOp(ISD::ABS,
-                       m_Sub(m_AllOf(m_Value(Op0), m_ZExt(m_VT(SrcVT0))),
-                             m_AllOf(m_Value(Op1), m_ZExt(m_VT(SrcVT1)))))) &&
+  return sd_match(Abs,
+                  m_Abs(m_Sub(m_AllOf(m_Value(Op0), m_ZExt(m_VT(SrcVT0))),
+                              m_AllOf(m_Value(Op1), m_ZExt(m_VT(SrcVT1)))))) &&
          SrcVT0.getVectorElementType() == MVT::i8 &&
          SrcVT1.getVectorElementType() == MVT::i8;
 }
-- 
cgit v1.2.3


From 7c15edb306932e41c159f3d69c161ed0d89d47b7 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Wed, 18 Jun 2025 14:37:29 +0200
Subject: =?UTF-8?q?Reapply=20"[clang][bytecode]=20Allocate=20IntegralAP=20?=
 =?UTF-8?q?and=20Floating=20types=20usi=E2=80=A6=20(#144676)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ng an allocator (#144246)"

This reverts commit 57828fec760f086b334ce0cb1c465fc559dcaea4.
---
 clang/lib/AST/ByteCode/Compiler.cpp                | 112 ++++---
 clang/lib/AST/ByteCode/Compiler.h                  |   1 +
 clang/lib/AST/ByteCode/Descriptor.cpp              |   2 +-
 clang/lib/AST/ByteCode/Disasm.cpp                  |  60 ++--
 clang/lib/AST/ByteCode/Floating.h                  | 252 ++++++++++-----
 clang/lib/AST/ByteCode/Integral.h                  |   3 +
 clang/lib/AST/ByteCode/IntegralAP.h                | 234 ++++++++------
 clang/lib/AST/ByteCode/Interp.cpp                  | 106 ++++++-
 clang/lib/AST/ByteCode/Interp.h                    | 337 +++++++++++++++++----
 clang/lib/AST/ByteCode/InterpBuiltin.cpp           |  55 +++-
 clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp    |   4 +-
 clang/lib/AST/ByteCode/InterpState.h               |  30 ++
 clang/lib/AST/ByteCode/Opcodes.td                  |  14 +-
 clang/lib/AST/ByteCode/PrimType.h                  |  17 ++
 clang/lib/AST/ByteCode/Program.h                   |  24 +-
 .../AST/ByteCode/builtin-bit-cast-long-double.cpp  |  10 +-
 clang/test/AST/ByteCode/builtin-functions.cpp      |  12 +-
 17 files changed, 930 insertions(+), 343 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 9fe4803ce98e..3f884ed8d094 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -748,7 +748,8 @@ bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
   if (DiscardResult)
     return true;
 
-  return this->emitConstFloat(E->getValue(), E);
+  APFloat F = E->getValue();
+  return this->emitFloat(F, E);
 }
 
 template <class Emitter>
@@ -4185,8 +4186,10 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
                              nullptr, E);
   case PT_MemberPtr:
     return this->emitNullMemberPtr(0, nullptr, E);
-  case PT_Float:
-    return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
+  case PT_Float: {
+    APFloat F = APFloat::getZero(Ctx.getFloatSemantics(QT));
+    return this->emitFloat(F, E);
+  }
   case PT_FixedPoint: {
     auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
     return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
@@ -4674,10 +4677,7 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       if (!visitInitializer(Init))
         return false;
 
-      if (!this->emitFinishInit(Init))
-        return false;
-
-      return this->emitPopPtr(Init);
+      return this->emitFinishInitGlobal(Init);
     };
 
     DeclScope<Emitter> LocalScope(this, VD);
@@ -4698,51 +4698,45 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       return false;
 
     return !Init || (checkDecl() && initGlobal(*GlobalIndex));
-  } else {
-    InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
-
-    if (VarT) {
-      unsigned Offset = this->allocateLocalPrimitive(
-          VD, *VarT, VD->getType().isConstQualified(), nullptr,
-          ScopeKind::Block, IsConstexprUnknown);
-      if (Init) {
-        // If this is a toplevel declaration, create a scope for the
-        // initializer.
-        if (Toplevel) {
-          LocalScope<Emitter> Scope(this);
-          if (!this->visit(Init))
-            return false;
-          return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
-        } else {
-          if (!this->visit(Init))
-            return false;
-          return this->emitSetLocal(*VarT, Offset, VD);
-        }
-      }
-    } else {
-      if (std::optional<unsigned> Offset =
-              this->allocateLocal(VD, VD->getType(), nullptr, ScopeKind::Block,
-                                  IsConstexprUnknown)) {
-        if (!Init)
-          return true;
+  }
+  // Local variables.
+  InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
 
-        if (!this->emitGetPtrLocal(*Offset, Init))
+  if (VarT) {
+    unsigned Offset = this->allocateLocalPrimitive(
+        VD, *VarT, VD->getType().isConstQualified(), nullptr, ScopeKind::Block,
+        IsConstexprUnknown);
+    if (Init) {
+      // If this is a toplevel declaration, create a scope for the
+      // initializer.
+      if (Toplevel) {
+        LocalScope<Emitter> Scope(this);
+        if (!this->visit(Init))
           return false;
-
-        if (!visitInitializer(Init))
+        return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
+      } else {
+        if (!this->visit(Init))
           return false;
+        return this->emitSetLocal(*VarT, Offset, VD);
+      }
+    }
+  } else {
+    if (std::optional<unsigned> Offset = this->allocateLocal(
+            VD, VD->getType(), nullptr, ScopeKind::Block, IsConstexprUnknown)) {
+      if (!Init)
+        return true;
 
-        if (!this->emitFinishInit(Init))
-          return false;
+      if (!this->emitGetPtrLocal(*Offset, Init))
+        return false;
 
-        return this->emitPopPtr(Init);
-      }
-      return false;
+      if (!visitInitializer(Init))
+        return false;
+
+      return this->emitFinishInitPop(Init);
     }
-    return true;
+    return false;
   }
-
-  return false;
+  return true;
 }
 
 template <class Emitter>
@@ -4751,8 +4745,10 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
   assert(!DiscardResult);
   if (Val.isInt())
     return this->emitConst(Val.getInt(), ValType, E);
-  else if (Val.isFloat())
-    return this->emitConstFloat(Val.getFloat(), E);
+  else if (Val.isFloat()) {
+    APFloat F = Val.getFloat();
+    return this->emitFloat(F, E);
+  }
 
   if (Val.isLValue()) {
     if (Val.isNullPointer())
@@ -6133,8 +6129,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+      APFloat F(TargetSemantics, 1);
+      if (!this->emitFloat(F, E))
         return false;
+
       if (!this->emitAddf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6176,8 +6174,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+      APFloat F(TargetSemantics, 1);
+      if (!this->emitFloat(F, E))
         return false;
+
       if (!this->emitSubf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6953,6 +6953,20 @@ bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
   return true;
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::emitFloat(const APFloat &F, const Expr *E) {
+  assert(!DiscardResult && "Should've been checked before");
+
+  if (Floating::singleWord(F.getSemantics()))
+    return this->emitConstFloat(Floating(F), E);
+
+  APInt I = F.bitcastToAPInt();
+  return this->emitConstFloat(
+      Floating(const_cast<uint64_t *>(I.getRawData()),
+               llvm::APFloatBase::SemanticsToEnum(F.getSemantics())),
+      E);
+}
+
 //  This function is constexpr if and only if To, From, and the types of
 //  all subobjects of To and From are types T such that...
 //  (3.1) - is_union_v<T> is false;
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index ac3ad84766dc..a1d068cc7e0a 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -391,6 +391,7 @@ private:
   bool emitRecordDestruction(const Record *R, SourceInfo Loc);
   bool emitDestruction(const Descriptor *Desc, SourceInfo Loc);
   bool emitDummyPtr(const DeclTy &D, const Expr *E);
+  bool emitFloat(const APFloat &F, const Expr *E);
   unsigned collectBaseOffset(const QualType BaseType,
                              const QualType DerivedType);
   bool emitLambdaStaticInvokerBody(const CXXMethodDecl *MD);
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 5531295dfa2f..46e4d0d940b3 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -368,7 +368,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsTemporary, bool IsConst, UnknownSize)
     : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
       MDSize(MD.value_or(0)),
-      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)),
+      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
       IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
       IsArray(true), CtorFn(getCtorArrayPrim(Type)),
       DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 846dc2fe92a7..7c6b78386b14 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -50,34 +50,56 @@ inline static std::string printArg(Program &P, CodePtr &OpPC) {
 }
 
 template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
-  auto F = Floating::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
+  auto Sem = Floating::deserializeSemantics(*OpPC);
 
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  unsigned BitWidth = llvm::APFloatBase::semanticsSizeInBits(
+      llvm::APFloatBase::EnumToSemantics(Sem));
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
+  Floating Result(Memory.get(), Sem);
+  Floating::deserialize(*OpPC, &Result);
+
+  OpPC += align(Result.bytesToSerialize());
+
+  std::string S;
+  llvm::raw_string_ostream SS(S);
+  SS << Result;
+  return S;
 }
 
 template <>
 inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
-  auto F = IntegralAP<false>::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
-
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  using T = IntegralAP<false>;
+  unsigned BitWidth = T::deserializeSize(*OpPC);
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
+
+  T Result(Memory.get(), BitWidth);
+  T::deserialize(*OpPC, &Result);
+
+  OpPC += Result.bytesToSerialize();
+  std::string Str;
+  llvm::raw_string_ostream SS(Str);
+  SS << Result;
+  return Str;
 }
+
 template <>
 inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
-  auto F = IntegralAP<true>::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
+  using T = IntegralAP<true>;
+  unsigned BitWidth = T::deserializeSize(*OpPC);
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
 
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  T Result(Memory.get(), BitWidth);
+  T::deserialize(*OpPC, &Result);
+
+  std::string Str;
+  llvm::raw_string_ostream SS(Str);
+  SS << Result;
+
+  OpPC += Result.bytesToSerialize();
+  return Str;
 }
 
 template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {
diff --git a/clang/lib/AST/ByteCode/Floating.h b/clang/lib/AST/ByteCode/Floating.h
index 3750568fc23c..659892e720ab 100644
--- a/clang/lib/AST/ByteCode/Floating.h
+++ b/clang/lib/AST/ByteCode/Floating.h
@@ -17,63 +17,79 @@
 #include "clang/AST/APValue.h"
 #include "llvm/ADT/APFloat.h"
 
+// XXX This is just a debugging help. Setting this to 1 will heap-allocate ALL
+// floating values.
+#define ALLOCATE_ALL 0
+
 namespace clang {
 namespace interp {
 
 using APFloat = llvm::APFloat;
 using APSInt = llvm::APSInt;
+using APInt = llvm::APInt;
 
+/// If a Floating is constructed from Memory, it DOES NOT OWN THAT MEMORY.
+/// It will NOT copy the memory (unless, of course, copy() is called) and it
+/// won't alllocate anything. The allocation should happen via InterpState or
+/// Program.
 class Floating final {
 private:
-  // The underlying value storage.
-  APFloat F;
+  union {
+    uint64_t Val = 0;
+    uint64_t *Memory;
+  };
+  llvm::APFloatBase::Semantics Semantics;
+
+  APFloat getValue() const {
+    unsigned BitWidth = bitWidth();
+    if (singleWord())
+      return APFloat(getSemantics(), APInt(BitWidth, Val));
+    unsigned NumWords = numWords();
+    return APFloat(getSemantics(), APInt(BitWidth, NumWords, Memory));
+  }
 
 public:
-  /// Zero-initializes a Floating.
-  Floating() : F(0.0f) {}
-  Floating(const APFloat &F) : F(F) {}
+  Floating() = default;
+  Floating(llvm::APFloatBase::Semantics Semantics)
+      : Val(0), Semantics(Semantics) {}
+  Floating(const APFloat &F) {
 
-  // Static constructors for special floating point values.
-  static Floating getInf(const llvm::fltSemantics &Sem) {
-    return Floating(APFloat::getInf(Sem));
+    Semantics = llvm::APFloatBase::SemanticsToEnum(F.getSemantics());
+    this->copy(F);
   }
-  const APFloat &getAPFloat() const { return F; }
+  Floating(uint64_t *Memory, llvm::APFloatBase::Semantics Semantics)
+      : Memory(Memory), Semantics(Semantics) {}
+
+  APFloat getAPFloat() const { return getValue(); }
 
-  bool operator<(Floating RHS) const { return F < RHS.F; }
-  bool operator>(Floating RHS) const { return F > RHS.F; }
-  bool operator<=(Floating RHS) const { return F <= RHS.F; }
-  bool operator>=(Floating RHS) const { return F >= RHS.F; }
-  bool operator==(Floating RHS) const { return F == RHS.F; }
-  bool operator!=(Floating RHS) const { return F != RHS.F; }
-  Floating operator-() const { return Floating(-F); }
+  bool operator<(Floating RHS) const { return getValue() < RHS.getValue(); }
+  bool operator>(Floating RHS) const { return getValue() > RHS.getValue(); }
+  bool operator<=(Floating RHS) const { return getValue() <= RHS.getValue(); }
+  bool operator>=(Floating RHS) const { return getValue() >= RHS.getValue(); }
 
   APFloat::opStatus convertToInteger(APSInt &Result) const {
     bool IsExact;
-    return F.convertToInteger(Result, llvm::APFloat::rmTowardZero, &IsExact);
+    return getValue().convertToInteger(Result, llvm::APFloat::rmTowardZero,
+                                       &IsExact);
   }
 
-  Floating toSemantics(const llvm::fltSemantics *Sem,
-                       llvm::RoundingMode RM) const {
-    APFloat Copy = F;
+  void toSemantics(const llvm::fltSemantics *Sem, llvm::RoundingMode RM,
+                   Floating *Result) const {
+    APFloat Copy = getValue();
     bool LosesInfo;
     Copy.convert(*Sem, RM, &LosesInfo);
     (void)LosesInfo;
-    return Floating(Copy);
-  }
-
-  /// Convert this Floating to one with the same semantics as \Other.
-  Floating toSemantics(const Floating &Other, llvm::RoundingMode RM) const {
-    return toSemantics(&Other.F.getSemantics(), RM);
+    Result->copy(Copy);
   }
 
   APSInt toAPSInt(unsigned NumBits = 0) const {
-    return APSInt(F.bitcastToAPInt());
+    return APSInt(getValue().bitcastToAPInt());
   }
-  APValue toAPValue(const ASTContext &) const { return APValue(F); }
+  APValue toAPValue(const ASTContext &) const { return APValue(getValue()); }
   void print(llvm::raw_ostream &OS) const {
     // Can't use APFloat::print() since it appends a newline.
     SmallVector<char, 16> Buffer;
-    F.toString(Buffer);
+    getValue().toString(Buffer);
     OS << Buffer;
   }
   std::string toDiagnosticString(const ASTContext &Ctx) const {
@@ -83,25 +99,62 @@ public:
     return NameStr;
   }
 
-  unsigned bitWidth() const { return F.semanticsSizeInBits(F.getSemantics()); }
+  unsigned bitWidth() const {
+    return llvm::APFloatBase::semanticsSizeInBits(getSemantics());
+  }
+  unsigned numWords() const { return llvm::APInt::getNumWords(bitWidth()); }
+  bool singleWord() const {
+#if ALLOCATE_ALL
+    return false;
+#endif
+    return numWords() == 1;
+  }
+  static bool singleWord(const llvm::fltSemantics &Sem) {
+#if ALLOCATE_ALL
+    return false;
+#endif
+    return APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem)) == 1;
+  }
+  const llvm::fltSemantics &getSemantics() const {
+    return llvm::APFloatBase::EnumToSemantics(Semantics);
+  }
+
+  void copy(const APFloat &F) {
+    if (singleWord()) {
+      Val = F.bitcastToAPInt().getZExtValue();
+    } else {
+      assert(Memory);
+      std::memcpy(Memory, F.bitcastToAPInt().getRawData(),
+                  numWords() * sizeof(uint64_t));
+    }
+  }
+
+  void take(uint64_t *NewMemory) {
+    if (singleWord())
+      return;
+
+    if (Memory)
+      std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
+    Memory = NewMemory;
+  }
 
   bool isSigned() const { return true; }
-  bool isNegative() const { return F.isNegative(); }
-  bool isZero() const { return F.isZero(); }
-  bool isNonZero() const { return F.isNonZero(); }
-  bool isMin() const { return F.isSmallest(); }
-  bool isMinusOne() const { return F.isExactlyValue(-1.0); }
-  bool isNan() const { return F.isNaN(); }
-  bool isSignaling() const { return F.isSignaling(); }
-  bool isInf() const { return F.isInfinity(); }
-  bool isFinite() const { return F.isFinite(); }
-  bool isNormal() const { return F.isNormal(); }
-  bool isDenormal() const { return F.isDenormal(); }
-  llvm::FPClassTest classify() const { return F.classify(); }
-  APFloat::fltCategory getCategory() const { return F.getCategory(); }
+  bool isNegative() const { return getValue().isNegative(); }
+  bool isZero() const { return getValue().isZero(); }
+  bool isNonZero() const { return getValue().isNonZero(); }
+  bool isMin() const { return getValue().isSmallest(); }
+  bool isMinusOne() const { return getValue().isExactlyValue(-1.0); }
+  bool isNan() const { return getValue().isNaN(); }
+  bool isSignaling() const { return getValue().isSignaling(); }
+  bool isInf() const { return getValue().isInfinity(); }
+  bool isFinite() const { return getValue().isFinite(); }
+  bool isNormal() const { return getValue().isNormal(); }
+  bool isDenormal() const { return getValue().isDenormal(); }
+  llvm::FPClassTest classify() const { return getValue().classify(); }
+  APFloat::fltCategory getCategory() const { return getValue().getCategory(); }
 
   ComparisonCategoryResult compare(const Floating &RHS) const {
-    llvm::APFloatBase::cmpResult CmpRes = F.compare(RHS.F);
+    llvm::APFloatBase::cmpResult CmpRes = getValue().compare(RHS.getValue());
     switch (CmpRes) {
     case llvm::APFloatBase::cmpLessThan:
       return ComparisonCategoryResult::Less;
@@ -118,97 +171,130 @@ public:
   static APFloat::opStatus fromIntegral(APSInt Val,
                                         const llvm::fltSemantics &Sem,
                                         llvm::RoundingMode RM,
-                                        Floating &Result) {
+                                        Floating *Result) {
     APFloat F = APFloat(Sem);
     APFloat::opStatus Status = F.convertFromAPInt(Val, Val.isSigned(), RM);
-    Result = Floating(F);
+    Result->copy(F);
     return Status;
   }
 
-  static Floating bitcastFromMemory(const std::byte *Buff,
-                                    const llvm::fltSemantics &Sem) {
+  static void bitcastFromMemory(const std::byte *Buff,
+                                const llvm::fltSemantics &Sem,
+                                Floating *Result) {
     size_t Size = APFloat::semanticsSizeInBits(Sem);
     llvm::APInt API(Size, true);
     llvm::LoadIntFromMemory(API, (const uint8_t *)Buff, Size / 8);
-
-    return Floating(APFloat(Sem, API));
+    Result->copy(APFloat(Sem, API));
   }
 
   void bitcastToMemory(std::byte *Buff) const {
-    llvm::APInt API = F.bitcastToAPInt();
+    llvm::APInt API = getValue().bitcastToAPInt();
     llvm::StoreIntToMemory(API, (uint8_t *)Buff, bitWidth() / 8);
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    return sizeof(llvm::fltSemantics *) +
-           (APFloat::semanticsSizeInBits(F.getSemantics()) / 8);
+    return sizeof(Semantics) + (numWords() * sizeof(uint64_t));
   }
 
   void serialize(std::byte *Buff) const {
-    // Semantics followed by an APInt.
-    *reinterpret_cast<const llvm::fltSemantics **>(Buff) = &F.getSemantics();
-
-    llvm::APInt API = F.bitcastToAPInt();
-    llvm::StoreIntToMemory(API, (uint8_t *)(Buff + sizeof(void *)),
-                           bitWidth() / 8);
+    std::memcpy(Buff, &Semantics, sizeof(Semantics));
+    if (singleWord()) {
+      std::memcpy(Buff + sizeof(Semantics), &Val, sizeof(uint64_t));
+    } else {
+      std::memcpy(Buff + sizeof(Semantics), Memory,
+                  numWords() * sizeof(uint64_t));
+    }
   }
 
-  static Floating deserialize(const std::byte *Buff) {
-    const llvm::fltSemantics *Sem;
-    std::memcpy((void *)&Sem, Buff, sizeof(void *));
-    return bitcastFromMemory(Buff + sizeof(void *), *Sem);
+  static llvm::APFloatBase::Semantics
+  deserializeSemantics(const std::byte *Buff) {
+    return *reinterpret_cast<const llvm::APFloatBase::Semantics *>(Buff);
   }
 
-  static Floating abs(const Floating &F) {
-    APFloat V = F.F;
-    if (V.isNegative())
-      V.changeSign();
-    return Floating(V);
+  static void deserialize(const std::byte *Buff, Floating *Result) {
+    llvm::APFloatBase::Semantics Semantics;
+    std::memcpy(&Semantics, Buff, sizeof(Semantics));
+
+    unsigned BitWidth = llvm::APFloat::semanticsSizeInBits(
+        llvm::APFloatBase::EnumToSemantics(Semantics));
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
+
+    Result->Semantics = Semantics;
+    if (NumWords == 1 && !ALLOCATE_ALL) {
+      std::memcpy(&Result->Val, Buff + sizeof(Semantics), sizeof(uint64_t));
+    } else {
+      assert(Result->Memory);
+      std::memcpy(Result->Memory, Buff + sizeof(Semantics),
+                  NumWords * sizeof(uint64_t));
+    }
   }
 
   // -------
 
   static APFloat::opStatus add(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.add(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.add(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus increment(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.F.getSemantics(), 1);
-    *R = Floating(A.F);
-    return R->F.add(One, RM);
+    APFloat One(A.getSemantics(), 1);
+    APFloat LHS = A.getValue();
+
+    auto Status = LHS.add(One, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus sub(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.subtract(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.subtract(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus decrement(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.F.getSemantics(), 1);
-    *R = Floating(A.F);
-    return R->F.subtract(One, RM);
+    APFloat One(A.getSemantics(), 1);
+    APFloat LHS = A.getValue();
+
+    auto Status = LHS.subtract(One, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus mul(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.multiply(B.F, RM);
+
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.multiply(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus div(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.divide(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.divide(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static bool neg(const Floating &A, Floating *R) {
-    *R = -A;
+    R->copy(-A.getValue());
     return false;
   }
 };
diff --git a/clang/lib/AST/ByteCode/Integral.h b/clang/lib/AST/ByteCode/Integral.h
index 13fdb5369f2b..af5cd2d13ecc 100644
--- a/clang/lib/AST/ByteCode/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -99,6 +99,9 @@ public:
   bool operator>=(Integral RHS) const { return V >= RHS.V; }
   bool operator==(Integral RHS) const { return V == RHS.V; }
   bool operator!=(Integral RHS) const { return V != RHS.V; }
+  bool operator>=(unsigned RHS) const {
+    return static_cast<unsigned>(V) >= RHS;
+  }
 
   bool operator>(unsigned RHS) const {
     return V >= 0 && static_cast<unsigned>(V) > RHS;
diff --git a/clang/lib/AST/ByteCode/IntegralAP.h b/clang/lib/AST/ByteCode/IntegralAP.h
index 8ee08dfb5cfe..61cbd14ad174 100644
--- a/clang/lib/AST/ByteCode/IntegralAP.h
+++ b/clang/lib/AST/ByteCode/IntegralAP.h
@@ -28,12 +28,19 @@ namespace interp {
 
 using APInt = llvm::APInt;
 using APSInt = llvm::APSInt;
-template <unsigned Bits, bool Signed> class Integral;
 
+/// If an IntegralAP is constructed from Memory, it DOES NOT OWN THAT MEMORY.
+/// It will NOT copy the memory (unless, of course, copy() is called) and it
+/// won't alllocate anything. The allocation should happen via InterpState or
+/// Program.
 template <bool Signed> class IntegralAP final {
-private:
+public:
+  union {
+    uint64_t *Memory = nullptr;
+    uint64_t Val;
+  };
+  uint32_t BitWidth = 0;
   friend IntegralAP<!Signed>;
-  APInt V;
 
   template <typename T, bool InputSigned>
   static T truncateCast(const APInt &V) {
@@ -52,106 +59,133 @@ private:
                                : V.trunc(BitSize).getZExtValue();
   }
 
+  APInt getValue() const {
+    if (singleWord())
+      return APInt(BitWidth, Val, Signed);
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
+    return llvm::APInt(BitWidth, NumWords, Memory);
+  }
+
 public:
   using AsUnsigned = IntegralAP<false>;
 
-  template <typename T>
-  IntegralAP(T Value, unsigned BitWidth)
-      : V(APInt(BitWidth, static_cast<uint64_t>(Value), Signed)) {}
+  void take(uint64_t *NewMemory) {
+    assert(!singleWord());
+    std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
+    Memory = NewMemory;
+  }
 
-  IntegralAP(APInt V) : V(V) {}
-  /// Arbitrary value for uninitialized variables.
-  IntegralAP() : IntegralAP(Signed ? -1 : 7, 3) {}
+  void copy(const APInt &V) {
+    assert(BitWidth == V.getBitWidth());
+    assert(numWords() == V.getNumWords());
 
-  IntegralAP operator-() const { return IntegralAP(-V); }
+    if (V.isSingleWord()) {
+      if constexpr (Signed)
+        Val = V.getSExtValue();
+      else
+        Val = V.getZExtValue();
+      return;
+    }
+    assert(Memory);
+    std::memcpy(Memory, V.getRawData(), V.getNumWords() * sizeof(uint64_t));
+  }
+
+  // Constructors.
+  IntegralAP() = default;
+  IntegralAP(unsigned BitWidth) : BitWidth(BitWidth) {}
+  IntegralAP(uint64_t *Memory, unsigned BitWidth)
+      : Memory(Memory), BitWidth(BitWidth) {}
+  IntegralAP(const APInt &V) : BitWidth(V.getBitWidth()) {
+    if (V.isSingleWord()) {
+      Val = Signed ? V.getSExtValue() : V.getZExtValue();
+    } else {
+      Memory = const_cast<uint64_t *>(V.getRawData());
+    }
+  }
+
+  IntegralAP operator-() const { return IntegralAP(-getValue()); }
   IntegralAP operator-(const IntegralAP &Other) const {
-    return IntegralAP(V - Other.V);
+    return IntegralAP(getValue() - Other.getValue());
   }
   bool operator>(const IntegralAP &RHS) const {
     if constexpr (Signed)
-      return V.ugt(RHS.V);
-    return V.sgt(RHS.V);
+      return getValue().sgt(RHS.getValue());
+    return getValue().ugt(RHS.getValue());
   }
-  bool operator>=(IntegralAP RHS) const {
+  bool operator>=(unsigned RHS) const {
     if constexpr (Signed)
-      return V.uge(RHS.V);
-    return V.sge(RHS.V);
+      return getValue().sge(RHS);
+    return getValue().uge(RHS);
   }
   bool operator<(IntegralAP RHS) const {
     if constexpr (Signed)
-      return V.slt(RHS.V);
-    return V.slt(RHS.V);
-  }
-  bool operator<=(IntegralAP RHS) const {
-    if constexpr (Signed)
-      return V.ult(RHS.V);
-    return V.ult(RHS.V);
+      return getValue().slt(RHS.getValue());
+    return getValue().ult(RHS.getValue());
   }
 
   template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
   explicit operator Ty() const {
-    return truncateCast<Ty, Signed>(V);
+    return truncateCast<Ty, Signed>(getValue());
   }
 
   template <typename T> static IntegralAP from(T Value, unsigned NumBits = 0) {
+    if (NumBits == 0)
+      NumBits = sizeof(T) * 8;
     assert(NumBits > 0);
     APInt Copy = APInt(NumBits, static_cast<uint64_t>(Value), Signed);
-
+    assert(false);
     return IntegralAP<Signed>(Copy);
   }
 
+  static IntegralAP from(const APInt &Value) {
+    return IntegralAP<Signed>(Value);
+  }
+
   template <bool InputSigned>
   static IntegralAP from(IntegralAP<InputSigned> V, unsigned NumBits = 0) {
     if (NumBits == 0)
       NumBits = V.bitWidth();
 
     if constexpr (InputSigned)
-      return IntegralAP<Signed>(V.V.sextOrTrunc(NumBits));
-    return IntegralAP<Signed>(V.V.zextOrTrunc(NumBits));
-  }
-
-  template <unsigned Bits, bool InputSigned>
-  static IntegralAP from(Integral<Bits, InputSigned> I, unsigned BitWidth) {
-    return IntegralAP<Signed>(I.toAPInt(BitWidth));
-  }
-
-  static IntegralAP zero(int32_t BitWidth) {
-    APInt V = APInt(BitWidth, 0LL, Signed);
-    return IntegralAP(V);
+      return IntegralAP<Signed>(V.getValue().sextOrTrunc(NumBits));
+    return IntegralAP<Signed>(V.getValue().zextOrTrunc(NumBits));
   }
 
-  constexpr unsigned bitWidth() const { return V.getBitWidth(); }
+  constexpr unsigned bitWidth() const { return BitWidth; }
+  constexpr unsigned numWords() const { return APInt::getNumWords(BitWidth); }
+  constexpr bool singleWord() const { return numWords() == 1; }
 
   APSInt toAPSInt(unsigned Bits = 0) const {
     if (Bits == 0)
       Bits = bitWidth();
 
+    APInt V = getValue();
     if constexpr (Signed)
-      return APSInt(V.sext(Bits), !Signed);
+      return APSInt(getValue().sext(Bits), !Signed);
     else
-      return APSInt(V.zext(Bits), !Signed);
+      return APSInt(getValue().zext(Bits), !Signed);
   }
   APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
-  bool isZero() const { return V.isZero(); }
+  bool isZero() const { return getValue().isZero(); }
   bool isPositive() const {
     if constexpr (Signed)
-      return V.isNonNegative();
+      return getValue().isNonNegative();
     return true;
   }
   bool isNegative() const {
     if constexpr (Signed)
-      return !V.isNonNegative();
+      return !getValue().isNonNegative();
     return false;
   }
-  bool isMin() const { return V.isMinValue(); }
-  bool isMax() const { return V.isMaxValue(); }
+  bool isMin() const { return getValue().isMinValue(); }
+  bool isMax() const { return getValue().isMaxValue(); }
   static constexpr bool isSigned() { return Signed; }
-  bool isMinusOne() const { return Signed && V == -1; }
+  bool isMinusOne() const { return Signed && getValue().isAllOnes(); }
 
-  unsigned countLeadingZeros() const { return V.countl_zero(); }
+  unsigned countLeadingZeros() const { return getValue().countl_zero(); }
 
-  void print(llvm::raw_ostream &OS) const { V.print(OS, Signed);}
+  void print(llvm::raw_ostream &OS) const { getValue().print(OS, Signed); }
   std::string toDiagnosticString(const ASTContext &Ctx) const {
     std::string NameStr;
     llvm::raw_string_ostream OS(NameStr);
@@ -161,53 +195,57 @@ public:
 
   IntegralAP truncate(unsigned BitWidth) const {
     if constexpr (Signed)
-      return IntegralAP(V.trunc(BitWidth).sextOrTrunc(this->bitWidth()));
+      return IntegralAP(
+          getValue().trunc(BitWidth).sextOrTrunc(this->bitWidth()));
     else
-      return IntegralAP(V.trunc(BitWidth).zextOrTrunc(this->bitWidth()));
+      return IntegralAP(
+          getValue().trunc(BitWidth).zextOrTrunc(this->bitWidth()));
   }
 
   IntegralAP<false> toUnsigned() const {
-    APInt Copy = V;
-    return IntegralAP<false>(Copy);
+    return IntegralAP<false>(Memory, BitWidth);
   }
 
   void bitcastToMemory(std::byte *Dest) const {
-    llvm::StoreIntToMemory(V, (uint8_t *)Dest, bitWidth() / 8);
+    llvm::StoreIntToMemory(getValue(), (uint8_t *)Dest, bitWidth() / 8);
   }
 
-  static IntegralAP bitcastFromMemory(const std::byte *Src, unsigned BitWidth) {
+  static void bitcastFromMemory(const std::byte *Src, unsigned BitWidth,
+                                IntegralAP *Result) {
     APInt V(BitWidth, static_cast<uint64_t>(0), Signed);
     llvm::LoadIntFromMemory(V, (const uint8_t *)Src, BitWidth / 8);
-    return IntegralAP(V);
+    Result->copy(V);
   }
 
   ComparisonCategoryResult compare(const IntegralAP &RHS) const {
     assert(Signed == RHS.isSigned());
     assert(bitWidth() == RHS.bitWidth());
+    APInt V1 = getValue();
+    APInt V2 = RHS.getValue();
     if constexpr (Signed) {
-      if (V.slt(RHS.V))
+      if (V1.slt(V2))
         return ComparisonCategoryResult::Less;
-      if (V.sgt(RHS.V))
+      if (V1.sgt(V2))
         return ComparisonCategoryResult::Greater;
       return ComparisonCategoryResult::Equal;
     }
 
     assert(!Signed);
-    if (V.ult(RHS.V))
+    if (V1.ult(V2))
       return ComparisonCategoryResult::Less;
-    if (V.ugt(RHS.V))
+    if (V1.ugt(V2))
       return ComparisonCategoryResult::Greater;
     return ComparisonCategoryResult::Equal;
   }
 
   static bool increment(IntegralAP A, IntegralAP *R) {
-    IntegralAP<Signed> One(1, A.bitWidth());
-    return add(A, One, A.bitWidth() + 1, R);
+    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
+    return add(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
   }
 
   static bool decrement(IntegralAP A, IntegralAP *R) {
-    IntegralAP<Signed> One(1, A.bitWidth());
-    return sub(A, One, A.bitWidth() + 1, R);
+    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
+    return sub(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
   }
 
   static bool add(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
@@ -224,87 +262,97 @@ public:
 
   static bool rem(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      *R = IntegralAP(A.V.srem(B.V));
+      R->copy(A.getValue().srem(B.getValue()));
     else
-      *R = IntegralAP(A.V.urem(B.V));
+      R->copy(A.getValue().urem(B.getValue()));
     return false;
   }
 
   static bool div(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      *R = IntegralAP(A.V.sdiv(B.V));
+      R->copy(A.getValue().sdiv(B.getValue()));
     else
-      *R = IntegralAP(A.V.udiv(B.V));
+      R->copy(A.getValue().udiv(B.getValue()));
     return false;
   }
 
   static bool bitAnd(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    *R = IntegralAP(A.V & B.V);
+    R->copy(A.getValue() & B.getValue());
     return false;
   }
 
   static bool bitOr(IntegralAP A, IntegralAP B, unsigned OpBits,
                     IntegralAP *R) {
-    *R = IntegralAP(A.V | B.V);
+    R->copy(A.getValue() | B.getValue());
     return false;
   }
 
   static bool bitXor(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    *R = IntegralAP(A.V ^ B.V);
+    R->copy(A.getValue() ^ B.getValue());
     return false;
   }
 
   static bool neg(const IntegralAP &A, IntegralAP *R) {
-    APInt AI = A.V;
+    APInt AI = A.getValue();
     AI.negate();
-    *R = IntegralAP(AI);
+    R->copy(AI);
     return false;
   }
 
   static bool comp(IntegralAP A, IntegralAP *R) {
-    *R = IntegralAP(~A.V);
+    R->copy(~A.getValue());
     return false;
   }
 
   static void shiftLeft(const IntegralAP A, const IntegralAP B, unsigned OpBits,
                         IntegralAP *R) {
-    *R = IntegralAP(A.V.shl(B.V.getZExtValue()));
+    *R = IntegralAP(A.getValue().shl(B.getValue().getZExtValue()));
   }
 
   static void shiftRight(const IntegralAP A, const IntegralAP B,
                          unsigned OpBits, IntegralAP *R) {
-    unsigned ShiftAmount = B.V.getZExtValue();
+    unsigned ShiftAmount = B.getValue().getZExtValue();
     if constexpr (Signed)
-      *R = IntegralAP(A.V.ashr(ShiftAmount));
+      R->copy(A.getValue().ashr(ShiftAmount));
     else
-      *R = IntegralAP(A.V.lshr(ShiftAmount));
+      R->copy(A.getValue().lshr(ShiftAmount));
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    // 4 bytes for the BitWidth followed by N bytes for the actual APInt.
-    return sizeof(uint32_t) + (V.getBitWidth() / CHAR_BIT);
+    assert(BitWidth != 0);
+    uint64_t NumWords = llvm::APInt::getNumWords(bitWidth());
+    return sizeof(uint32_t) + (NumWords * sizeof(uint64_t));
   }
 
   void serialize(std::byte *Buff) const {
-    assert(V.getBitWidth() < std::numeric_limits<uint8_t>::max());
-    uint32_t BitWidth = V.getBitWidth();
-
     std::memcpy(Buff, &BitWidth, sizeof(uint32_t));
-    llvm::StoreIntToMemory(V, (uint8_t *)(Buff + sizeof(uint32_t)),
-                           BitWidth / CHAR_BIT);
+    if (singleWord())
+      std::memcpy(Buff + sizeof(uint32_t), &Val, sizeof(uint64_t));
+    else {
+      uint64_t NumWords = llvm::APInt::getNumWords(bitWidth());
+      std::memcpy(Buff + sizeof(uint32_t), Memory, NumWords * sizeof(uint64_t));
+    }
   }
 
-  static IntegralAP<Signed> deserialize(const std::byte *Buff) {
-    uint32_t BitWidth;
-    std::memcpy(&BitWidth, Buff, sizeof(uint32_t));
-    IntegralAP<Signed> Val(APInt(BitWidth, 0ull, !Signed));
+  static uint64_t deserializeSize(const std::byte *Buff) {
+    return *reinterpret_cast<const uint32_t *>(Buff);
+  }
+
+  static void deserialize(const std::byte *Buff, IntegralAP<Signed> *Result) {
+    uint64_t BitWidth = Result->BitWidth;
+    uint64_t NumWords = llvm::APInt::getNumWords(BitWidth);
+    assert(BitWidth == Result->BitWidth);
 
-    llvm::LoadIntFromMemory(Val.V, (const uint8_t *)Buff + sizeof(uint32_t),
-                            BitWidth / CHAR_BIT);
-    return Val;
+    if (NumWords == 1)
+      std::memcpy(&Result->Val, Buff + sizeof(uint32_t), sizeof(uint64_t));
+    else {
+      assert(Result->Memory);
+      std::memcpy(Result->Memory, Buff + sizeof(uint32_t),
+                  NumWords * sizeof(uint64_t));
+    }
   }
 
 private:
@@ -312,7 +360,7 @@ private:
   static bool CheckAddSubMulUB(const IntegralAP &A, const IntegralAP &B,
                                unsigned BitWidth, IntegralAP *R) {
     if constexpr (!Signed) {
-      R->V = Op<APInt>{}(A.V, B.V);
+      R->copy(Op<APInt>{}(A.getValue(), B.getValue()));
       return false;
     }
 
@@ -320,7 +368,7 @@ private:
     const APSInt &RHS = B.toAPSInt();
     APSInt Value = Op<APSInt>{}(LHS.extend(BitWidth), RHS.extend(BitWidth));
     APSInt Result = Value.trunc(LHS.getBitWidth());
-    R->V = Result;
+    R->copy(Result);
 
     return Result.extend(BitWidth) != Value;
   }
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 5c8abffb3a99..1e2032feabb6 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1935,8 +1935,10 @@ bool CastPointerIntegralAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  S.Stk.push<IntegralAP<false>>(
-      IntegralAP<false>::from(Ptr.getIntegerRepresentation(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
+
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
@@ -1946,8 +1948,10 @@ bool CastPointerIntegralAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  S.Stk.push<IntegralAP<true>>(
-      IntegralAP<true>::from(Ptr.getIntegerRepresentation(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
+
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2053,6 +2057,100 @@ bool arePotentiallyOverlappingStringLiterals(const Pointer &LHS,
   return Shorter == Longer.take_front(Shorter.size());
 }
 
+static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr,
+                                PrimType T) {
+
+  if (T == PT_IntAPS) {
+    auto &Val = Ptr.deref<IntegralAP<true>>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  } else if (T == PT_IntAP) {
+    auto &Val = Ptr.deref<IntegralAP<false>>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  } else if (T == PT_Float) {
+    auto &Val = Ptr.deref<Floating>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  }
+}
+
+template <typename T>
+static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr) {
+  assert(needsAlloc<T>());
+  auto &Val = Ptr.deref<T>();
+  if (!Val.singleWord()) {
+    uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+    Val.take(NewMemory);
+  }
+}
+
+static void finishGlobalRecurse(InterpState &S, const Pointer &Ptr) {
+  if (const Record *R = Ptr.getRecord()) {
+    for (const Record::Field &Fi : R->fields()) {
+      if (Fi.Desc->isPrimitive()) {
+        TYPE_SWITCH_ALLOC(Fi.Desc->getPrimType(), {
+          copyPrimitiveMemory<T>(S, Ptr.atField(Fi.Offset));
+        });
+        copyPrimitiveMemory(S, Ptr.atField(Fi.Offset), Fi.Desc->getPrimType());
+      } else
+        finishGlobalRecurse(S, Ptr.atField(Fi.Offset));
+    }
+    return;
+  }
+
+  if (const Descriptor *D = Ptr.getFieldDesc(); D && D->isArray()) {
+    unsigned NumElems = D->getNumElems();
+    if (NumElems == 0)
+      return;
+
+    if (D->isPrimitiveArray()) {
+      PrimType PT = D->getPrimType();
+      if (!needsAlloc(PT))
+        return;
+      assert(NumElems >= 1);
+      const Pointer EP = Ptr.atIndex(0);
+      bool AllSingleWord = true;
+      TYPE_SWITCH_ALLOC(PT, {
+        if (!EP.deref<T>().singleWord()) {
+          copyPrimitiveMemory<T>(S, EP);
+          AllSingleWord = false;
+        }
+      });
+      if (AllSingleWord)
+        return;
+      for (unsigned I = 1; I != D->getNumElems(); ++I) {
+        const Pointer EP = Ptr.atIndex(I);
+        copyPrimitiveMemory(S, EP, PT);
+      }
+    } else {
+      assert(D->isCompositeArray());
+      for (unsigned I = 0; I != D->getNumElems(); ++I) {
+        const Pointer EP = Ptr.atIndex(I).narrow();
+        finishGlobalRecurse(S, EP);
+      }
+    }
+  }
+}
+
+bool FinishInitGlobal(InterpState &S, CodePtr OpPC) {
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+
+  finishGlobalRecurse(S, Ptr);
+  if (Ptr.canBeInitialized()) {
+    Ptr.initialize();
+    Ptr.activate();
+  }
+
+  return true;
+}
+
 // https://github.com/llvm/llvm-project/issues/102513
 #if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
 #pragma optimize("", off)
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index ae3d4a441a79..66d3e6d79e8b 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -189,7 +189,7 @@ bool CheckShift(InterpState &S, CodePtr OpPC, const LT &LHS, const RT &RHS,
 
   // C++11 [expr.shift]p1: Shift width must be less than the bit width of
   // the shifted type.
-  if (Bits > 1 && RHS >= RT::from(Bits, RHS.bitWidth())) {
+  if (Bits > 1 && RHS >= Bits) {
     const Expr *E = S.Current->getExpr(OpPC);
     const APSInt Val = RHS.toAPSInt();
     QualType Ty = E->getType();
@@ -370,6 +370,9 @@ bool AddSubMulHelper(InterpState &S, CodePtr OpPC, unsigned Bits, const T &LHS,
                      const T &RHS) {
   // Fast path - add the numbers with fixed width.
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!OpFW(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -408,6 +411,7 @@ bool Add(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
+
   return AddSubMulHelper<T, T::add, std::plus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -423,7 +427,7 @@ inline bool Addf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::add(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -434,6 +438,7 @@ bool Sub(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
+
   return AddSubMulHelper<T, T::sub, std::minus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -442,7 +447,7 @@ inline bool Subf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::sub(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -453,6 +458,7 @@ bool Mul(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() * 2;
+
   return AddSubMulHelper<T, T::mul, std::multiplies>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -461,8 +467,10 @@ inline bool Mulf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
+
   auto Status = Floating::mul(LHS, RHS, getRoundingMode(FPO), &Result);
+
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -484,9 +492,14 @@ inline bool Mulc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexMul(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    Floating RA = S.allocFloat(A.getSemantics());
+    RA.copy(ResR);
+    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
     Result.atIndex(0).initialize();
-    Result.atIndex(1).deref<Floating>() = Floating(ResI);
+
+    Floating RI = S.allocFloat(A.getSemantics());
+    RI.copy(ResI);
+    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
     Result.atIndex(1).initialize();
     Result.initialize();
   } else {
@@ -539,10 +552,20 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    // Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    // Result.atIndex(0).initialize();
+    // Result.atIndex(1).deref<Floating>() = Floating(ResI);
+    // Result.atIndex(1).initialize();
+
+    Floating RA = S.allocFloat(A.getSemantics());
+    RA.copy(ResR);
+    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
     Result.atIndex(0).initialize();
-    Result.atIndex(1).deref<Floating>() = Floating(ResI);
-    Result.atIndex(1).initialize();
+
+    Floating RI = S.allocFloat(A.getSemantics());
+    RI.copy(ResI);
+    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
+
     Result.initialize();
   } else {
     // Integer element type.
@@ -608,9 +631,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitAnd(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitAnd(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -625,9 +651,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitOr(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitOr(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -644,7 +673,11 @@ bool BitXor(InterpState &S, CodePtr OpPC) {
   const T &LHS = S.Stk.pop<T>();
 
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitXor(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -659,12 +692,15 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Rem(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
+  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
-  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!T::rem(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -679,12 +715,15 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Div(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
+  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
-  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!T::div(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -707,8 +746,10 @@ inline bool Divf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
     return false;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::div(LHS, RHS, getRoundingMode(FPO), &Result);
+
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -730,31 +771,44 @@ inline bool Inv(InterpState &S, CodePtr OpPC) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Neg(InterpState &S, CodePtr OpPC) {
   const T &Value = S.Stk.pop<T>();
-  T Result;
 
-  if (!T::neg(Value, &Result)) {
+  if constexpr (std::is_same_v<T, Floating>) {
+    T Result = S.allocFloat(Value.getSemantics());
+
+    if (!T::neg(Value, &Result)) {
+      S.Stk.push<T>(Result);
+      return true;
+    }
+    return false;
+  } else {
+    T Result;
+    if constexpr (needsAlloc<T>())
+      Result = S.allocAP<T>(Value.bitWidth());
+
+    if (!T::neg(Value, &Result)) {
+      S.Stk.push<T>(Result);
+      return true;
+    }
+
+    assert(isIntegralType(Name) &&
+           "don't expect other types to fail at constexpr negation");
     S.Stk.push<T>(Result);
-    return true;
-  }
 
-  assert(isIntegralType(Name) &&
-         "don't expect other types to fail at constexpr negation");
-  S.Stk.push<T>(Result);
+    APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
+    if (S.checkingForUndefinedBehavior()) {
+      const Expr *E = S.Current->getExpr(OpPC);
+      QualType Type = E->getType();
+      SmallString<32> Trunc;
+      NegatedValue.trunc(Result.bitWidth())
+          .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
+                    /*UpperCase=*/true, /*InsertSeparators=*/true);
+      S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
+          << Trunc << Type << E->getSourceRange();
+      return true;
+    }
 
-  APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
-  if (S.checkingForUndefinedBehavior()) {
-    const Expr *E = S.Current->getExpr(OpPC);
-    QualType Type = E->getType();
-    SmallString<32> Trunc;
-    NegatedValue.trunc(Result.bitWidth())
-        .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
-                  /*UpperCase=*/true, /*InsertSeparators=*/true);
-    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
-        << Trunc << Type << E->getSourceRange();
-    return true;
+    return handleOverflow(S, OpPC, NegatedValue);
   }
-
-  return handleOverflow(S, OpPC, NegatedValue);
 }
 
 enum class PushVal : bool {
@@ -783,6 +837,8 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
   const T &Value = Ptr.deref<T>();
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Value.bitWidth());
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<T>(Value);
@@ -890,7 +946,6 @@ bool PreDec(InterpState &S, CodePtr OpPC, bool CanOverflow) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckLoad(S, OpPC, Ptr, AK_Decrement))
     return false;
-
   return IncDecHelper<T, IncDecOp::Dec, PushVal::No>(S, OpPC, Ptr, CanOverflow);
 }
 
@@ -898,7 +953,7 @@ template <IncDecOp Op, PushVal DoPush>
 bool IncDecFloatHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                        uint32_t FPOI) {
   Floating Value = Ptr.deref<Floating>();
-  Floating Result;
+  Floating Result = S.allocFloat(Value.getSemantics());
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<Floating>(Value);
@@ -952,12 +1007,15 @@ inline bool DecfPop(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Comp(InterpState &S, CodePtr OpPC) {
   const T &Val = S.Stk.pop<T>();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Val.bitWidth());
+
   if (!T::comp(Val, &Result)) {
     S.Stk.push<T>(Result);
     return true;
   }
-
   return false;
 }
 
@@ -1325,10 +1383,23 @@ bool Flip(InterpState &S, CodePtr OpPC) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Const(InterpState &S, CodePtr OpPC, const T &Arg) {
+  if constexpr (needsAlloc<T>()) {
+    T Result = S.allocAP<T>(Arg.bitWidth());
+    Result.copy(Arg.toAPSInt());
+    S.Stk.push<T>(Result);
+    return true;
+  }
   S.Stk.push<T>(Arg);
   return true;
 }
 
+inline bool ConstFloat(InterpState &S, CodePtr OpPC, const Floating &F) {
+  Floating Result = S.allocFloat(F.getSemantics());
+  Result.copy(F.getAPFloat());
+  S.Stk.push<Floating>(Result);
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Get/Set Local/Param/Global/This
 //===----------------------------------------------------------------------===//
@@ -1483,7 +1554,24 @@ bool SetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool InitGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &P = S.P.getGlobal(I);
+
   P.deref<T>() = S.Stk.pop<T>();
+
+  if constexpr (std::is_same_v<T, Floating>) {
+    auto &Val = P.deref<Floating>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+
+  } else if constexpr (needsAlloc<T>()) {
+    auto &Val = P.deref<T>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  }
+
   P.initialize();
   return true;
 }
@@ -1585,7 +1673,22 @@ bool InitBitField(InterpState &S, CodePtr OpPC, const Record::Field *F) {
   assert(F->isBitField());
   const T &Value = S.Stk.pop<T>();
   const Pointer &Field = S.Stk.peek<Pointer>().atField(F->Offset);
-  Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
+
+  if constexpr (needsAlloc<T>()) {
+    T Result = S.allocAP<T>(Value.bitWidth());
+    if (T::isSigned())
+      Result.copy(Value.toAPSInt()
+                      .trunc(F->Decl->getBitWidthValue())
+                      .sextOrTrunc(Value.bitWidth()));
+    else
+      Result.copy(Value.toAPSInt()
+                      .trunc(F->Decl->getBitWidthValue())
+                      .zextOrTrunc(Value.bitWidth()));
+
+    Field.deref<T>() = Result;
+  } else {
+    Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
+  }
   Field.activate();
   Field.initialize();
   return true;
@@ -1765,6 +1868,8 @@ inline bool FinishInit(InterpState &S, CodePtr OpPC) {
   return true;
 }
 
+bool FinishInitGlobal(InterpState &S, CodePtr OpPC);
+
 inline bool Dump(InterpState &S, CodePtr OpPC) {
   S.Stk.dump();
   return true;
@@ -2271,7 +2376,8 @@ template <PrimType TIn, PrimType TOut> bool Cast(InterpState &S, CodePtr OpPC) {
 inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
                    llvm::RoundingMode RM) {
   Floating F = S.Stk.pop<Floating>();
-  Floating Result = F.toSemantics(Sem, RM);
+  Floating Result = S.allocFloat(*Sem);
+  F.toSemantics(Sem, RM, &Result);
   S.Stk.push<Floating>(Result);
   return true;
 }
@@ -2295,15 +2401,25 @@ inline bool CastFixedPoint(InterpState &S, CodePtr OpPC, uint32_t FPS) {
 /// to know what bitwidth the result should be.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<false>>(
-      IntegralAP<false>::from(S.Stk.pop<T>(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  // Copy data.
+  {
+    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
+    Result.copy(Source);
+  }
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<true>>(
-      IntegralAP<true>::from(S.Stk.pop<T>(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  // Copy data.
+  {
+    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
+    Result.copy(Source);
+  }
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2312,11 +2428,11 @@ bool CastIntegralFloating(InterpState &S, CodePtr OpPC,
                           const llvm::fltSemantics *Sem, uint32_t FPOI) {
   const T &From = S.Stk.pop<T>();
   APSInt FromAP = From.toAPSInt();
-  Floating Result;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
+  Floating Result = S.allocFloat(*Sem);
   auto Status =
-      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), Result);
+      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
 
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -2365,7 +2481,12 @@ static inline bool CastFloatingIntegralAP(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+
+  auto ResultAP = S.allocAP<IntegralAP<false>>(BitWidth);
+  ResultAP.copy(Result);
+
+  S.Stk.push<IntegralAP<false>>(ResultAP);
+
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2381,7 +2502,12 @@ static inline bool CastFloatingIntegralAPS(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+
+  auto ResultAP = S.allocAP<IntegralAP<true>>(BitWidth);
+  ResultAP.copy(Result);
+
+  S.Stk.push<IntegralAP<true>>(ResultAP);
+
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2441,8 +2567,9 @@ static inline bool CastFloatingFixedPoint(InterpState &S, CodePtr OpPC,
 static inline bool CastFixedPointFloating(InterpState &S, CodePtr OpPC,
                                           const llvm::fltSemantics *Sem) {
   const auto &Fixed = S.Stk.pop<FixedPoint>();
-
-  S.Stk.push<Floating>(Fixed.toFloat(Sem));
+  Floating Result = S.allocFloat(*Sem);
+  Result.copy(Fixed.toFloat(Sem));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -2506,12 +2633,18 @@ bool Zero(InterpState &S, CodePtr OpPC) {
 }
 
 static inline bool ZeroIntAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<false>>(IntegralAP<false>::zero(BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  if (!Result.singleWord())
+    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
 static inline bool ZeroIntAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>::zero(BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  if (!Result.singleWord())
+    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2578,7 +2711,9 @@ inline bool RVOPtr(InterpState &S, CodePtr OpPC) {
 //===----------------------------------------------------------------------===//
 
 template <class LT, class RT, ShiftDir Dir>
-inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
+inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
+                    LT *Result) {
+
   const unsigned Bits = LHS.bitWidth();
 
   // OpenCL 6.3j: shift values are effectively % word size of LHS.
@@ -2596,7 +2731,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
     RHS = -RHS;
     return DoShift<LT, RT,
                    Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
-        S, OpPC, LHS, RHS);
+        S, OpPC, LHS, RHS, Result);
   }
 
   if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
@@ -2644,6 +2779,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
       // Do the shift on potentially signed LT, then convert to unsigned type.
       LT A;
       LT::shiftRight(LHS, LT::from(RHS, Bits), Bits, &A);
+      // LT::shiftRight(LHS, LT(RHSTemp), Bits, &A);
       R = LT::AsUnsigned::from(A);
     }
   }
@@ -2652,6 +2788,48 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
   return true;
 }
 
+/// A version of DoShift that works on IntegralAP.
+template <class LT, class RT, ShiftDir Dir>
+inline bool DoShiftAP(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
+                      LT *Result) {
+  const unsigned Bits = LHS.bitWidth();
+  const APSInt &LHSAP = LHS.toAPSInt();
+  APSInt RHSAP = RHS.toAPSInt();
+
+  // OpenCL 6.3j: shift values are effectively % word size of LHS.
+  if (S.getLangOpts().OpenCL)
+    RHSAP &= APSInt(llvm::APInt(RHSAP.getBitWidth(),
+                                static_cast<uint64_t>(LHSAP.getBitWidth() - 1)),
+                    RHSAP.isUnsigned());
+
+  if (RHS.isNegative()) {
+    // During constant-folding, a negative shift is an opposite shift. Such a
+    // shift is not a constant expression.
+    const SourceInfo &Loc = S.Current->getSource(OpPC);
+    S.CCEDiag(Loc, diag::note_constexpr_negative_shift) << RHS.toAPSInt();
+    if (!S.noteUndefinedBehavior())
+      return false;
+    RHS = -RHS;
+    return DoShiftAP<LT, RT,
+                     Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
+        S, OpPC, LHS, RHS, Result);
+  }
+
+  if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
+    return false;
+
+  if constexpr (Dir == ShiftDir::Left) {
+    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
+    Result->copy(LHSAP << SA);
+  } else {
+    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
+    Result->copy(LHSAP >> SA);
+  }
+
+  S.Stk.push<LT>(*Result);
+  return true;
+}
+
 template <PrimType NameL, PrimType NameR>
 inline bool Shr(InterpState &S, CodePtr OpPC) {
   using LT = typename PrimConv<NameL>::T;
@@ -2659,7 +2837,13 @@ inline bool Shr(InterpState &S, CodePtr OpPC) {
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
 
-  return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS);
+  if constexpr (needsAlloc<LT>()) {
+    LT Result = S.allocAP<LT>(LHS.bitWidth());
+    return DoShiftAP<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
+  } else {
+    LT Result;
+    return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
+  }
 }
 
 template <PrimType NameL, PrimType NameR>
@@ -2668,8 +2852,13 @@ inline bool Shl(InterpState &S, CodePtr OpPC) {
   using RT = typename PrimConv<NameR>::T;
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
-
-  return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS);
+  if constexpr (needsAlloc<LT>()) {
+    LT Result = S.allocAP<LT>(LHS.bitWidth());
+    return DoShiftAP<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
+  } else {
+    LT Result;
+    return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
+  }
 }
 
 static inline bool ShiftFixedPoint(InterpState &S, CodePtr OpPC, bool Left) {
@@ -3252,7 +3441,15 @@ inline bool BitCastPrim(InterpState &S, CodePtr OpPC, bool TargetIsUCharOrByte,
 
     if constexpr (std::is_same_v<T, Floating>) {
       assert(Sem);
-      S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
+      Floating Result = S.allocFloat(*Sem);
+      Floating::bitcastFromMemory(Buff.data(), *Sem, &Result);
+      S.Stk.push<Floating>(Result);
+
+      // S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
+    } else if constexpr (needsAlloc<T>()) {
+      T Result = S.allocAP<T>(ResultBitWidth);
+      T::bitcastFromMemory(Buff.data(), ResultBitWidth, &Result);
+      S.Stk.push<T>(Result);
     } else {
       assert(!Sem);
       S.Stk.push<T>(T::bitcastFromMemory(Buff.data(), ResultBitWidth));
@@ -3310,7 +3507,11 @@ template <typename T> inline T ReadArg(InterpState &S, CodePtr &OpPC) {
 }
 
 template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
-  Floating F = Floating::deserialize(*OpPC);
+  auto &Semantics =
+      llvm::APFloatBase::EnumToSemantics(Floating::deserializeSemantics(*OpPC));
+
+  auto F = S.allocFloat(Semantics);
+  Floating::deserialize(*OpPC, &F);
   OpPC += align(F.bytesToSerialize());
   return F;
 }
@@ -3318,17 +3519,25 @@ template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
 template <>
 inline IntegralAP<false> ReadArg<IntegralAP<false>>(InterpState &S,
                                                     CodePtr &OpPC) {
-  IntegralAP<false> I = IntegralAP<false>::deserialize(*OpPC);
-  OpPC += align(I.bytesToSerialize());
-  return I;
+  uint32_t BitWidth = IntegralAP<false>::deserializeSize(*OpPC);
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  assert(Result.bitWidth() == BitWidth);
+
+  IntegralAP<false>::deserialize(*OpPC, &Result);
+  OpPC += align(Result.bytesToSerialize());
+  return Result;
 }
 
 template <>
 inline IntegralAP<true> ReadArg<IntegralAP<true>>(InterpState &S,
                                                   CodePtr &OpPC) {
-  IntegralAP<true> I = IntegralAP<true>::deserialize(*OpPC);
-  OpPC += align(I.bytesToSerialize());
-  return I;
+  uint32_t BitWidth = IntegralAP<true>::deserializeSize(*OpPC);
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  assert(Result.bitWidth() == BitWidth);
+
+  IntegralAP<true>::deserialize(*OpPC, &Result);
+  OpPC += align(Result.bytesToSerialize());
+  return Result;
 }
 
 template <>
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index d01e3d042a8b..5304bd77f2c0 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -57,6 +57,21 @@ static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
   assert(T);
 
   unsigned BitWidth = S.getASTContext().getTypeSize(QT);
+
+  if (T == PT_IntAPS) {
+    auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+    Result.copy(Val);
+    S.Stk.push<IntegralAP<true>>(Result);
+    return;
+  }
+
+  if (T == PT_IntAP) {
+    auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+    Result.copy(Val);
+    S.Stk.push<IntegralAP<false>>(Result);
+    return;
+  }
+
   if (QT->isSignedIntegerOrEnumerationType()) {
     int64_t V = Val.getSExtValue();
     INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
@@ -327,13 +342,13 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  Floating Result;
+  Floating Result = S.allocFloat(TargetSemantics);
   if (S.getASTContext().getTargetInfo().isNan2008()) {
     if (Signaling)
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
   } else {
     // Prior to IEEE 754-2008, architectures were allowed to choose whether
@@ -342,10 +357,10 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
     // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
     // sNaN. This is now known as "legacy NaN" encoding.
     if (Signaling)
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
   }
 
@@ -360,7 +375,9 @@ static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  S.Stk.push<Floating>(Floating::getInf(TargetSemantics));
+  Floating Result = S.allocFloat(TargetSemantics);
+  Result.copy(APFloat::getInf(TargetSemantics));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -368,10 +385,12 @@ static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame) {
   const Floating &Arg2 = S.Stk.pop<Floating>();
   const Floating &Arg1 = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(Arg1.getSemantics());
 
   APFloat Copy = Arg1.getAPFloat();
   Copy.copySign(Arg2.getAPFloat());
-  S.Stk.push<Floating>(Floating(Copy));
+  Result.copy(Copy);
+  S.Stk.push<Floating>(Result);
 
   return true;
 }
@@ -380,11 +399,13 @@ static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    S.Stk.push<Floating>(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    S.Stk.push<Floating>(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -392,11 +413,13 @@ static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    S.Stk.push<Floating>(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    S.Stk.push<Floating>(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -571,8 +594,16 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame) {
   const Floating &Val = S.Stk.pop<Floating>();
+  APFloat F = Val.getAPFloat();
+  if (!F.isNegative()) {
+    S.Stk.push<Floating>(Val);
+    return true;
+  }
 
-  S.Stk.push<Floating>(Floating::abs(Val));
+  Floating Result = S.allocFloat(Val.getSemantics());
+  F.changeSign();
+  Result.copy(F);
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 239b3104e89f..2569cac018b3 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -402,7 +402,9 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
           if (llvm::sys::IsBigEndianHost)
             swapBytes(M.get(), NumBits.roundToBytes());
 
-          P.deref<Floating>() = Floating::bitcastFromMemory(M.get(), Semantics);
+          Floating R = S.allocFloat(Semantics);
+          Floating::bitcastFromMemory(M.get(), Semantics, &R);
+          P.deref<Floating>() = R;
           P.initialize();
           return true;
         }
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index e8dc6f0483d6..08765561985e 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -15,6 +15,7 @@
 
 #include "Context.h"
 #include "DynamicAllocator.h"
+#include "Floating.h"
 #include "Function.h"
 #include "InterpFrame.h"
 #include "InterpStack.h"
@@ -126,6 +127,33 @@ public:
 
   StdAllocatorCaller getStdAllocatorCaller(StringRef Name) const;
 
+  void *allocate(size_t Size, unsigned Align = 8) const {
+    return Allocator.Allocate(Size, Align);
+  }
+  template <typename T> T *allocate(size_t Num = 1) const {
+    return static_cast<T *>(allocate(Num * sizeof(T), alignof(T)));
+  }
+
+  template <typename T> T allocAP(unsigned BitWidth) {
+    unsigned NumWords = APInt::getNumWords(BitWidth);
+    if (NumWords == 1)
+      return T(BitWidth);
+    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
+    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
+    return T(Mem, BitWidth);
+  }
+
+  Floating allocFloat(const llvm::fltSemantics &Sem) {
+    if (Floating::singleWord(Sem))
+      return Floating(llvm::APFloatBase::SemanticsToEnum(Sem));
+
+    unsigned NumWords =
+        APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem));
+    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
+    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
+    return Floating(Mem, llvm::APFloatBase::SemanticsToEnum(Sem));
+  }
+
 private:
   friend class EvaluationResult;
   friend class InterpStateCCOverride;
@@ -161,6 +189,8 @@ public:
   llvm::SmallVector<
       std::pair<const Expr *, const LifetimeExtendedTemporaryDecl *>>
       SeenGlobalTemporaries;
+
+  mutable llvm::BumpPtrAllocator Allocator;
 };
 
 class InterpStateCCOverride final {
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index c76ac5f8ae86..57e01f7bd9da 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -48,6 +48,7 @@ def ArgUint64 : ArgType { let Name = "uint64_t"; }
 def ArgIntAP : ArgType { let Name = "IntegralAP<false>"; let AsRef = true; }
 def ArgIntAPS : ArgType { let Name = "IntegralAP<true>"; let AsRef = true; }
 def ArgFloat : ArgType { let Name = "Floating"; let AsRef = true; }
+
 def ArgBool : ArgType { let Name = "bool"; }
 def ArgFixedPoint : ArgType { let Name = "FixedPoint"; let AsRef = true; }
 
@@ -88,6 +89,9 @@ def IntegerAndFixedTypeClass : TypeClass {
                Uint32, Sint64, Uint64, IntAP, IntAPS, FixedPoint];
 }
 
+def IntegralTypeClass : TypeClass {
+  let Types = !listconcat(IntegerTypeClass.Types, [Bool]);
+}
 def FixedSizeIntegralTypeClass : TypeClass {
   let Types = [Sint8, Uint8, Sint16, Uint16, Sint32,
                Uint32, Sint64, Uint64, Bool];
@@ -265,12 +269,13 @@ def ConstSint32 : ConstOpcode<Sint32, ArgSint32>;
 def ConstUint32 : ConstOpcode<Uint32, ArgUint32>;
 def ConstSint64 : ConstOpcode<Sint64, ArgSint64>;
 def ConstUint64 : ConstOpcode<Uint64, ArgUint64>;
-def ConstFloat : ConstOpcode<Float, ArgFloat>;
-def constIntAP : ConstOpcode<IntAP, ArgIntAP>;
-def constIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
+def ConstIntAP : ConstOpcode<IntAP, ArgIntAP>;
+def ConstIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
 def ConstBool : ConstOpcode<Bool, ArgBool>;
 def ConstFixedPoint : ConstOpcode<FixedPoint, ArgFixedPoint>;
 
+def ConstFloat : Opcode { let Args = [ArgFloat]; }
+
 // [] -> [Integer]
 def Zero : Opcode {
   let Types = [FixedSizeIntegralTypeClass];
@@ -328,6 +333,7 @@ def GetMemberPtrBasePop : Opcode {
 
 def FinishInitPop : Opcode;
 def FinishInit    : Opcode;
+def FinishInitGlobal : Opcode;
 
 def GetPtrDerivedPop : Opcode { let Args = [ArgUint32, ArgBool, ArgTypePtr]; }
 
@@ -389,7 +395,7 @@ class AccessOpcode : Opcode {
 }
 
 class BitFieldOpcode : Opcode {
-  let Types = [AluTypeClass];
+  let Types = [IntegralTypeClass];
   let Args = [ArgRecordField];
   let HasGroup = 1;
 }
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index 6152fbfbe3a7..a156cccbb3c1 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -76,6 +76,13 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
 }
 
 constexpr bool isIntegralType(PrimType T) { return T <= PT_FixedPoint; }
+template <typename T> constexpr bool needsAlloc() {
+  return std::is_same_v<T, IntegralAP<false>> ||
+         std::is_same_v<T, IntegralAP<true>> || std::is_same_v<T, Floating>;
+}
+constexpr bool needsAlloc(PrimType T) {
+  return T == PT_IntAP || T == PT_IntAPS || T == PT_Float;
+}
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
@@ -209,6 +216,16 @@ static inline bool aligned(const void *P) {
     }                                                                          \
   } while (0)
 
+#define TYPE_SWITCH_ALLOC(Expr, B)                                             \
+  do {                                                                         \
+    switch (Expr) {                                                            \
+      TYPE_SWITCH_CASE(PT_Float, B)                                            \
+      TYPE_SWITCH_CASE(PT_IntAP, B)                                            \
+      TYPE_SWITCH_CASE(PT_IntAPS, B)                                           \
+    default:;                                                                  \
+    }                                                                          \
+  } while (0)
+
 #define COMPOSITE_TYPE_SWITCH(Expr, B, D)                                      \
   do {                                                                         \
     switch (Expr) {                                                            \
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index 23ba1bbd193b..5d9c42244749 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -132,6 +132,14 @@ public:
                                bool IsMutable = false, bool IsVolatile = false,
                                const Expr *Init = nullptr);
 
+  void *Allocate(size_t Size, unsigned Align = 8) const {
+    return Allocator.Allocate(Size, Align);
+  }
+  template <typename T> T *Allocate(size_t Num = 1) const {
+    return static_cast<T *>(Allocate(Num * sizeof(T), alignof(T)));
+  }
+  void Deallocate(void *Ptr) const {}
+
   /// Context to manage declaration lifetimes.
   class DeclScope {
   public:
@@ -204,7 +212,7 @@ private:
   };
 
   /// Allocator for globals.
-  PoolAllocTy Allocator;
+  mutable PoolAllocTy Allocator;
 
   /// Global objects.
   std::vector<Global *> Globals;
@@ -238,4 +246,18 @@ public:
 } // namespace interp
 } // namespace clang
 
+inline void *operator new(size_t Bytes, const clang::interp::Program &C,
+                          size_t Alignment = 8) {
+  return C.Allocate(Bytes, Alignment);
+}
+
+inline void operator delete(void *Ptr, const clang::interp::Program &C,
+                            size_t) {
+  C.Deallocate(Ptr);
+}
+inline void *operator new[](size_t Bytes, const clang::interp::Program &C,
+                            size_t Alignment = 8) {
+  return C.Allocate(Bytes, Alignment);
+}
+
 #endif
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
index 710612bef8fd..1013a771d13b 100644
--- a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
+++ b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
@@ -21,6 +21,9 @@ template <class To, class From>
 constexpr To bit_cast(const From &from) {
   static_assert(sizeof(To) == sizeof(From));
   return __builtin_bit_cast(To, from);
+#if __x86_64
+  // both-note@-2 {{indeterminate value can only initialize an object of type}}
+#endif
 }
 
 template <class Intermediate, class Init>
@@ -38,11 +41,8 @@ constexpr Init round_trip(const Init &init) {
 
 namespace test_long_double {
 #if __x86_64
-/// FIXME: We could enable this, but since it aborts, it causes the usual mempory leak.
-#if 0
-constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // expected-error{{must be initialized by a constant expression}}\
-                                                                                 // expected-note{{in call}}
-#endif
+constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // both-error{{must be initialized by a constant expression}}\
+                                                                                 // both-note{{in call}}
 constexpr long double ld = 3.1425926539;
 
 struct bytes {
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 21dca15a4577..174c1ffa79a4 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -208,7 +208,7 @@ namespace nan {
 
   constexpr double NaN3 = __builtin_nan("foo"); // both-error {{must be initialized by a constant expression}}
   constexpr float NaN4 = __builtin_nanf("");
-  //constexpr long double NaN5 = __builtin_nanf128("");
+  constexpr long double NaN5 = __builtin_nanf128("");
 
   /// FIXME: This should be accepted by the current interpreter as well.
   constexpr char f[] = {'0', 'x', 'A', 'E', '\0'};
@@ -655,8 +655,6 @@ void test_noexcept(int *i) {
 } // end namespace test_launder
 
 
-/// FIXME: The commented out tests here use a IntAP value and fail.
-/// This currently means we will leak the IntAP value since nothing cleans it up.
 namespace clz {
   char clz1[__builtin_clz(1) == BITSIZE(int) - 1 ? 1 : -1];
   char clz2[__builtin_clz(7) == BITSIZE(int) - 3 ? 1 : -1];
@@ -709,7 +707,7 @@ namespace clz {
   char clz48[__builtin_clzg(1ULL << (BITSIZE(long long) - 1)) == 0 ? 1 : -1];
   char clz49[__builtin_clzg(1ULL << (BITSIZE(long long) - 1), 42) == 0 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  // int clz50 = __builtin_clzg((unsigned __int128)0);
+  int clz50 = __builtin_clzg((unsigned __int128)0);
   char clz51[__builtin_clzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char clz52[__builtin_clzg((unsigned __int128)0x1) == BITSIZE(__int128) - 1 ? 1 : -1];
   char clz53[__builtin_clzg((unsigned __int128)0x1, 42) == BITSIZE(__int128) - 1 ? 1 : -1];
@@ -717,7 +715,7 @@ namespace clz {
   char clz55[__builtin_clzg((unsigned __int128)0xf, 42) == BITSIZE(__int128) - 4 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  // int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
+  int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
   char clz59[__builtin_clzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char clz60[__builtin_clzg((unsigned _BitInt(128))0x1) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
   char clz61[__builtin_clzg((unsigned _BitInt(128))0x1, 42) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
@@ -775,7 +773,7 @@ namespace ctz {
   char ctz46[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1)) == BITSIZE(long long) - 1 ? 1 : -1];
   char ctz47[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1), 42) == BITSIZE(long long) - 1 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  // int ctz48 = __builtin_ctzg((unsigned __int128)0);
+  int ctz48 = __builtin_ctzg((unsigned __int128)0);
   char ctz49[__builtin_ctzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char ctz50[__builtin_ctzg((unsigned __int128)0x1) == 0 ? 1 : -1];
   char ctz51[__builtin_ctzg((unsigned __int128)0x1, 42) == 0 ? 1 : -1];
@@ -785,7 +783,7 @@ namespace ctz {
   char ctz55[__builtin_ctzg((unsigned __int128)1 << (BITSIZE(__int128) - 1), 42) == BITSIZE(__int128) - 1 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  // int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
+  int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
   char ctz57[__builtin_ctzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char ctz58[__builtin_ctzg((unsigned _BitInt(128))0x1) == 0 ? 1 : -1];
   char ctz59[__builtin_ctzg((unsigned _BitInt(128))0x1, 42) == 0 ? 1 : -1];
-- 
cgit v1.2.3


From 6265ca686dfe18e6032e59637f144bad7ea6cf2b Mon Sep 17 00:00:00 2001
From: Ties Stuij <ties.stuij@arm.com>
Date: Wed, 18 Jun 2025 13:38:49 +0100
Subject: [AArch64] Add Cortex-A320 scheduling model (#144385)

Instead of using the Cortex-A510 scheduling model, Cortex-A320 now uses
its own scheduling model, based off of the Cortex-A320 Software
Optimization Guide:

https://developer.arm.com/documentation/110285/r0p1

---------

Co-authored-by: Nashe Mncube <Nashe.Mncube@arm.com>
---
 llvm/lib/Target/AArch64/AArch64.td                 |     1 +
 llvm/lib/Target/AArch64/AArch64Processors.td       |     2 +-
 llvm/lib/Target/AArch64/AArch64SchedA320.td        |  1415 +++
 .../AArch64/Cortex/A320-basic-instructions.s       |  3721 +++++++
 .../AArch64/Cortex/A320-neon-instructions.s        |  3208 ++++++
 .../AArch64/Cortex/A320-sve-instructions.s         | 10258 +++++++++++++++++++
 6 files changed, 18604 insertions(+), 1 deletion(-)
 create mode 100644 llvm/lib/Target/AArch64/AArch64SchedA320.td
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A320-basic-instructions.s
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A320-neon-instructions.s
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A320-sve-instructions.s

diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index f303819f411d..eb5a5199b895 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -103,6 +103,7 @@ def MTEUnsupported : AArch64Unsupported {
 let F = [HasPAuth, HasPAuthLR] in
 def PAUnsupported : AArch64Unsupported;
 
+include "AArch64SchedA320.td"
 include "AArch64SchedA53.td"
 include "AArch64SchedA55.td"
 include "AArch64SchedA510.td"
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index c7ea6393e2ad..e1b82953aad8 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -1131,7 +1131,7 @@ def : ProcessorModel<"cortex-a34", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA35]>;
 def : ProcessorModel<"cortex-a35", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA35]>;
-def : ProcessorModel<"cortex-a320", CortexA510Model, ProcessorFeatures.A320,
+def : ProcessorModel<"cortex-a320", CortexA320Model, ProcessorFeatures.A320,
                      [TuneA320]>;
 def : ProcessorModel<"cortex-a53", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA53]>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA320.td b/llvm/lib/Target/AArch64/AArch64SchedA320.td
new file mode 100644
index 000000000000..89ed13389daf
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64SchedA320.td
@@ -0,0 +1,1415 @@
+//==- AArch64SchedCortexA320.td - ARM Cortex-A320 Scheduling Definitions -*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the machine model for the ARM Cortex-A320 processor.
+//
+//===----------------------------------------------------------------------===//
+
+// ===---------------------------------------------------------------------===//
+// The following definitions describe the per-operand machine model.
+// This works with MachineScheduler. See MCSchedModel.h for details.
+
+// Cortex-A320 machine model for scheduling and other instruction cost heuristics.
+def CortexA320Model : SchedMachineModel {
+  let MicroOpBufferSize = 0;  // Cortex-A320 is an in-order processor
+  let IssueWidth = 1;         // Cortex-A320 is a single-issue processor
+  let LoadLatency = 5;
+  let PostRAScheduler = 1;    // Enable PostRA scheduler pass.
+  let CompleteModel = 0;      // Covers instructions applicable to Cortex-A320.
+
+  let FullInstRWOverlapCheck = 0;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Subtarget-specific SchedWrite types
+
+let SchedModel = CortexA320Model in {
+
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available.
+
+// Modeling each pipeline as a ProcResource using the BufferSize = 0 since the
+// Cortex-A320 is in-order.
+let BufferSize = 0 in {
+  def CortexA320UnitALU    : ProcResource<1>;    // Int ALU
+  def CortexA320UnitMAC    : ProcResource<1>;    // Int MAC, 64-bit wide
+  def CortexA320UnitDiv    : ProcResource<1>;    // Int Division, not pipelined
+  def CortexA320UnitLdSt   : ProcResource<1>;    // Load/Store shared pipe
+  def CortexA320UnitB      : ProcResource<1>;    // Branch
+  def CortexA320UnitPAC    : ProcResource<1>;    // Pointer Authentication (PAC) pipe
+
+  // The FP DIV/SQRT instructions execute totally differently from the FP ALU
+  // instructions; that's why for now we model them with 2 resources.
+  def CortexA320UnitVALU  : ProcResource<1>;    // SIMD/FP/SVE ALU
+  def CortexA320UnitVMAC   : ProcResource<1>;    // SIMD/FP/SVE MAC
+  def CortexA320UnitVMC    : ProcResource<1>;    // SIMD/FP/SVE multicycle instrs  (e.g Div, SQRT, cryptography)
+}
+
+// These latencies are modeled without taking into account forwarding paths
+// (the software optimisation guide lists latencies taking into account
+// typical forwarding paths).
+def : WriteRes<WriteImm,   [CortexA320UnitALU]> { let Latency = 1; }  // MOVN, MOVZ
+def : WriteRes<WriteI,     [CortexA320UnitALU]> { let Latency = 1; }  // ALU
+def : WriteRes<WriteISReg, [CortexA320UnitALU]> { let Latency = 2; }  // ALU of Shifted-Reg
+def : WriteRes<WriteIEReg, [CortexA320UnitALU]> { let Latency = 2; }  // ALU of Extended-Reg
+def : WriteRes<WriteExtr, [CortexA320UnitALU]> { let Latency = 2; }   // EXTR from a reg pair
+def : WriteRes<WriteIS, [CortexA320UnitALU]> { let Latency = 2; }     // Shift/Scale
+
+// MAC
+def : WriteRes<WriteIM32, [CortexA320UnitMAC]> { let Latency = 3; }   // 32-bit Multiply
+def : WriteRes<WriteIM64, [CortexA320UnitMAC]> { let Latency = 5; let ReleaseAtCycles = [2];}   // 64-bit Multiply
+
+// Div
+def : WriteRes<WriteID32, [CortexA320UnitDiv]> {
+  let Latency = 12; let ReleaseAtCycles = [12];
+}
+def : WriteRes<WriteID64, [CortexA320UnitDiv]> {
+  let Latency = 20; let ReleaseAtCycles = [20];
+}
+
+//===----------------------------------------------------------------------===//
+// Define customized scheduler read/write types specific to Cortex-A320
+
+//===----------------------------------------------------------------------===//
+class CortexA320Write<int n, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+}
+
+class CortexA320MCWrite<int n, int m, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+  let ReleaseAtCycles = [m];
+  let BeginGroup = 1;
+}
+
+class CortexA320MC_RC0Write<int n, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+  let BeginGroup = 1;
+}
+
+//===----------------------------------------------------------------------===//
+
+// Define generic 2 micro-op types
+def CortexA320Write_11cyc_1VMAC_1VALU : SchedWriteRes<[CortexA320UnitVALU, CortexA320UnitVMAC]> {
+  let Latency     = 11;
+  let NumMicroOps = 2;
+}
+
+def CortexA320Write_16cyc_1VMAC_1VALU : SchedWriteRes<[CortexA320UnitVALU, CortexA320UnitVMAC]> {
+  let Latency     = 16;
+  let NumMicroOps = 2;
+}
+
+class CortexA320Write_PAC_B <int lat> : SchedWriteRes<[CortexA320UnitPAC, CortexA320UnitB]> {
+  let Latency = lat;
+  let NumMicroOps = 2;
+}
+
+// Load
+def : WriteRes<WriteLD, [CortexA320UnitLdSt]> { let Latency = 4; }
+def : WriteRes<WriteLDIdx, [CortexA320UnitLdSt]> { let Latency = 4; }
+def : WriteRes<WriteLDHi, [CortexA320UnitLdSt]> { let Latency = 4; }
+
+def CortexA320WriteVLD1 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 3; }
+def CortexA320WriteVLD1SI : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 3; let SingleIssue = 1; }
+
+def CortexA320WriteVLD2 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4;
+                                                  let ReleaseAtCycles = [2]; }
+
+def CortexA320WriteVLD3 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5;
+                                                  let ReleaseAtCycles = [3]; }
+
+def CortexA320WriteVLD4 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 6;
+                                                  let ReleaseAtCycles = [4]; }
+
+def CortexA320WriteVLD6 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5;
+                                                  let ReleaseAtCycles = [3]; }
+
+def CortexA320WriteVLD8 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 6;
+                                                  let ReleaseAtCycles = [4]; }
+
+def CortexA320WriteLDP1 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; }
+def CortexA320WriteLDP2 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; }
+def CortexA320WriteLDP4 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; }
+
+// Pre/Post Indexing - Performed as part of address generation
+def : WriteRes<WriteAdr, []> { let Latency = 0; }
+
+// Store
+let RetireOOO = 1 in {
+def : WriteRes<WriteST, [CortexA320UnitLdSt]> { let Latency = 1; }
+def : WriteRes<WriteSTP, [CortexA320UnitLdSt]> { let Latency = 1; }
+def : WriteRes<WriteSTIdx, [CortexA320UnitLdSt]> { let Latency = 1; }
+}
+def : WriteRes<WriteSTX, [CortexA320UnitLdSt]> { let Latency = 3; }
+
+// Vector Store - Similar to vector loads, can take 1-3 cycles to issue.
+def : WriteRes<WriteVST, [CortexA320UnitLdSt]> { let Latency = 5;
+                                          let ReleaseAtCycles = [2];}
+def CortexA320WriteVST1 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; }
+def CortexA320WriteVST2 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5;
+                                                  let ReleaseAtCycles = [2]; }
+def CortexA320WriteVST3 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5;
+                                                  let ReleaseAtCycles = [3]; }
+def CortexA320WriteVST4 : SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5;
+                                                  let ReleaseAtCycles = [4]; }
+
+def : WriteRes<WriteAtomic, []> { let Unsupported = 1; }
+
+// Branch
+def : WriteRes<WriteBr, [CortexA320UnitB]>;
+def : WriteRes<WriteBrReg, [CortexA320UnitB]>;
+def : WriteRes<WriteSys, [CortexA320UnitB]>;
+def : WriteRes<WriteBarrier, [CortexA320UnitB]>;
+def : WriteRes<WriteHint, [CortexA320UnitB]>;
+
+// FP ALU
+//   As WriteF result is produced in F5 and it can be mostly forwarded
+//   to consumer at F1, the effectively Latency is set as 4.
+def : WriteRes<WriteF, [CortexA320UnitVALU]> { let Latency = 4; }
+def : WriteRes<WriteFCmp, [CortexA320UnitVALU]> { let Latency = 3; }
+def : WriteRes<WriteFCvt, [CortexA320UnitVALU]> { let Latency = 4; }
+def : WriteRes<WriteFCopy, [CortexA320UnitVALU]> { let Latency = 3; }
+def : WriteRes<WriteFImm, [CortexA320UnitVALU]> { let Latency = 3; }
+
+class CortexA320VSt<int n> : SchedWriteRes<[CortexA320UnitLdSt]> {
+  let RetireOOO = 1;
+  let ReleaseAtCycles = [n];
+}
+
+def CortexA320VSt0      : SchedWriteRes<[CortexA320UnitLdSt]> {
+  let RetireOOO = 1;
+}
+
+def : SchedAlias<WriteVd, CortexA320Write<4, CortexA320UnitVALU>>;
+def : SchedAlias<WriteVq, CortexA320Write<4, CortexA320UnitVALU>>;
+
+// FP ALU specific new schedwrite definitions
+def CortexA320WriteFPALU_F3 : SchedWriteRes<[CortexA320UnitVALU]> { let Latency = 3;}
+def CortexA320WriteFPALU_F4 : SchedWriteRes<[CortexA320UnitVALU]> { let Latency = 4;}
+
+// FP Mul, Div, Sqrt. Div/Sqrt are not pipelined
+def : WriteRes<WriteFMul, [CortexA320UnitVMAC]> { let Latency = 4; }
+
+let RetireOOO = 1 in {
+def : WriteRes<WriteFDiv, [CortexA320UnitVMC]> { let Latency = 22;
+                                            let ReleaseAtCycles = [29]; }
+def CortexA320WriteVMAC : SchedWriteRes<[CortexA320UnitVMAC]> { let Latency = 4; }
+def CortexA320WriteFDivHP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 8;
+                                                     let ReleaseAtCycles = [5]; }
+def CortexA320WriteFDivSP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 13;
+                                                     let ReleaseAtCycles = [10]; }
+def CortexA320WriteFDivDP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 22;
+                                                     let ReleaseAtCycles = [19]; }
+def CortexA320WriteFSqrtHP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 8;
+                                                      let ReleaseAtCycles = [5]; }
+def CortexA320WriteFSqrtSP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 12;
+                                                      let ReleaseAtCycles = [9]; }
+def CortexA320WriteFSqrtDP : SchedWriteRes<[CortexA320UnitVMC]> { let Latency = 22;
+                                                      let ReleaseAtCycles = [19]; }
+}
+
+//===----------------------------------------------------------------------===//
+// Subtarget-specific SchedRead types.
+
+def : ReadAdvance<ReadVLD, 0>;
+def : ReadAdvance<ReadExtrHi, 0>;
+def : ReadAdvance<ReadAdrBase, 0>;
+def : ReadAdvance<ReadST, 1>;
+
+def : ReadAdvance<ReadI, 0>;
+def : ReadAdvance<ReadISReg, 0>;
+def : ReadAdvance<ReadIEReg, 0>;
+
+
+// MUL
+def : ReadAdvance<ReadIM, 0>;
+def : ReadAdvance<ReadIMA, 2>;
+
+// Div
+def : ReadAdvance<ReadID, 0>;
+
+//===----------------------------------------------------------------------===//
+// Subtarget-specific InstRWs.
+
+def CortexA320WriteISReg : SchedWriteVariant<[
+       SchedVar<RegShiftedPred, [WriteISReg]>,
+       SchedVar<NoSchedPred, [WriteI]>]>;
+def : InstRW<[CortexA320WriteISReg], (instregex ".*rs$")>;
+def : InstRW<[WriteIS], (instrs RBITWr, RBITXr)>;
+
+// Pointer Authentication Instructions (v8.3 PAC)
+// -----------------------------------------------------------------------------
+
+// Authenticate data address
+// Authenticate instruction address
+// Compute pointer authentication code for data address
+// Compute pointer authentication code, using generic key
+// Compute pointer authentication code for instruction address
+def : InstRW<[CortexA320Write<4, CortexA320UnitPAC>], (instregex "^AUT", "^PAC")>;
+
+// Branch and link, register, with pointer authentication
+// Branch, register, with pointer authentication
+// Branch, return, with pointer authentication
+def : InstRW<[CortexA320Write_PAC_B<1>], (instrs BLRAA, BLRAAZ, BLRAB, BLRABZ, BRAA,
+                                            BRAAZ, BRAB, BRABZ, RETAA, RETAB,
+                                            ERETAA, ERETAB)>;
+
+// Load register, with pointer authentication
+def : InstRW<[CortexA320Write<2, CortexA320UnitPAC>], (instregex "^LDRA[AB](indexed|writeback)")>;
+
+// Strip pointer authentication code
+def : InstRW<[CortexA320Write<5, CortexA320UnitPAC>], (instrs XPACD, XPACI, XPACLRI)>;
+//---
+// Miscellaneous
+//---
+def : InstRW<[CortexA320WriteVLD1SI,CortexA320WriteLDP1], (instregex "LDPS?Wi")>;
+def : InstRW<[CortexA320WriteVLD1,CortexA320WriteLDP1], (instregex "LDPSi")>;
+def : InstRW<[CortexA320WriteVLD1,CortexA320WriteLDP2], (instregex "LDP(X|D)i")>;
+def : InstRW<[CortexA320WriteVLD1,CortexA320WriteLDP4], (instregex "LDPQi")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1SI,CortexA320WriteLDP1], (instregex "LDPS?W(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1,CortexA320WriteLDP1], (instregex "LDPS(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1,CortexA320WriteLDP2], (instregex "LDP(X|D)(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1,CortexA320WriteLDP4], (instregex "LDPQ(pre|post)")>;
+def : InstRW<[WriteI], (instrs COPY)>;
+//---
+// Vector Loads - 128-bit per cycle
+//---
+//   1-element structures
+def CortexA320WriteVLD1Latency3: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 3; let ReleaseAtCycles = [1]; }
+def CortexA320WriteVLD1Latency4: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [2]; }
+def CortexA320WriteVLD1Latency5: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5; let ReleaseAtCycles = [3]; }
+def CortexA320WriteVLD1Latency6: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 6; let ReleaseAtCycles = [4]; }
+
+def : InstRW<[CortexA320WriteVLD1Latency3], (instregex "LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency3], (instregex "LD1Twov(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency4], (instregex "LD1Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency4], (instregex "LD1Threev(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency5], (instregex "LD1Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency4], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency6], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVLD1Latency3], (instregex "LD1i(8|16|32|64)$")>;                // single element
+def : InstRW<[CortexA320WriteVLD1Latency3], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // replicate
+
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency3], (instregex "LD1Onev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency3], (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency4], (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency4], (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency5], (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency4], (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency6], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency3], (instregex "LD1i(8|16|32|64)_POST$")>;                // single element
+def : InstRW<[WriteAdr, CortexA320WriteVLD1Latency3], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>; // replicate
+
+//    2-element structures
+def CortexA320WriteVLD2Latency3: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 3; let ReleaseAtCycles = [2]; }
+def CortexA320WriteVLD2Latency4Release1: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [1]; }
+def CortexA320WriteVLD2Latency4Release2: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [2]; }
+def CortexA320WriteVLD2Latency4Release6: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [6]; }
+
+def : InstRW<[CortexA320WriteVLD2Latency4Release1], (instregex "LD2Twov(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVLD2Latency4Release2], (instregex "LD2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVLD2Latency4Release6], (instregex "LD2i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVLD2Latency3], (instregex "LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
+
+def : InstRW<[WriteAdr, CortexA320WriteVLD2Latency4Release1], (instregex "LD2Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD2Latency4Release2], (instregex "LD2Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD2Latency4Release6], (instregex "LD2i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD2Latency3], (instregex "LD2Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
+
+//    3-element structures
+def CortexA320WriteVLD3Latency4: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [3]; }
+def CortexA320WriteVLD3Latency5Release6: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5; let ReleaseAtCycles = [6]; }
+def CortexA320WriteVLD3Latency5Release7: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5; let ReleaseAtCycles = [7]; }
+
+def : InstRW<[CortexA320WriteVLD3Latency5Release6], (instregex "LD3Threev(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
+def : InstRW<[CortexA320WriteVLD3Latency5Release7], (instregex "LD3i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVLD3Latency4], (instregex "LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)$")>;
+
+def : InstRW<[WriteAdr, CortexA320WriteVLD3Latency5Release6], (instregex "LD3Threev(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD3Latency5Release7], (instregex "LD3i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD3Latency4], (instregex "LD3Rv(8b|16b|4h|8h|2s|4s|1d|2d)_POST$")>;
+
+//    4-element structures
+def CortexA320WriteVLD4Latency4: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 4; let ReleaseAtCycles = [4]; }
+def CortexA320WriteVLD4Latency5Release7: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5; let ReleaseAtCycles = [7]; }
+def CortexA320WriteVLD4Latency5Release8: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 5; let ReleaseAtCycles = [8]; }
+def CortexA320WriteVLD4Latency6: SchedWriteRes<[CortexA320UnitLdSt]> { let Latency = 6; let ReleaseAtCycles = [7]; }
+
+def : InstRW<[CortexA320WriteVLD4Latency5Release7], (instregex "LD4Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVLD4Latency5Release8], (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVLD4Latency6], (instregex "LD4i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVLD4Latency4], (instregex "LD4Rv(8b|16b|4h|8b|2s|4s|1d|2d)$")>;
+ 
+def : InstRW<[WriteAdr, CortexA320WriteVLD4Latency5Release7], (instregex "LD4Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD4Latency5Release8], (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>; 
+def : InstRW<[WriteAdr, CortexA320WriteVLD4Latency6], (instregex "LD4i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVLD4Latency4], (instregex "LD4Rv(8b|16b|4h|8b|2s|4s|1d|2d)_POST$")>;
+//---
+// Vector Stores
+//---
+// 1 Element structures
+def : InstRW<[CortexA320WriteVST1], (instregex "ST1i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVST1], (instregex "ST1Onev(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVST2], (instregex "ST1Onev(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVST1], (instregex "ST1Twov(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA320WriteVST2], (instregex "ST1Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVST2], (instregex "ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA320WriteVST4], (instregex "ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST1], (instregex "ST1i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST1], (instregex "ST1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST1], (instregex "ST1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST4], (instregex "ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+// 2 Element structures
+def : InstRW<[CortexA320WriteVST2], (instregex "ST2i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVST2], (instregex "ST2Twov(8b|4h|2s)$")>;
+def : InstRW<[CortexA320WriteVST4], (instregex "ST2Twov(16b|8h|4s|2d)$")>;
+
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST2i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST4], (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
+
+// 3 Element structures
+def : InstRW<[CortexA320WriteVST2], (instregex "ST3i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVST4], (instregex "ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST3i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST4], (instregex "ST3Threev(8b|4h|2s|1d|2d|16b|8h|4s|4d)_POST$")>;
+
+// 4 Element structures
+def : InstRW<[CortexA320WriteVST2], (instregex "ST4i(8|16|32|64)$")>;
+def : InstRW<[CortexA320WriteVST4], (instregex "ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+
+def : InstRW<[WriteAdr, CortexA320WriteVST2], (instregex "ST4i(8|16|32|64)_POST$")>;
+def : InstRW<[WriteAdr, CortexA320WriteVST4], (instregex "ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+//---
+// Floating Point Conversions, MAC, DIV, SQRT
+//---
+def : InstRW<[CortexA320WriteFPALU_F3], (instregex "^DUP(v2i64|v4i32|v8i16|v16i8)")>;
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^XTN")>;
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^FCVT[ALMNPZ][SU](S|U)?(W|X)")>;
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^FCVT(X)?[ALMNPXZ](S|U|N)?v")>;
+
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^(S|U)CVTF(S|U)(W|X)(H|S|D)")>;
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^(S|U)CVTF(h|s|d)")>;
+def : InstRW<[CortexA320WriteFPALU_F4], (instregex "^(S|U)CVTFv")>;
+
+def : InstRW<[CortexA320WriteVMAC], (instregex "^FN?M(ADD|SUB).*")>;
+def : InstRW<[CortexA320WriteVMAC], (instregex "^FML(A|S)v.*")>;
+def : InstRW<[CortexA320WriteFDivHP], (instrs FDIVHrr)>;
+def : InstRW<[CortexA320WriteFDivSP], (instrs FDIVSrr)>;
+def : InstRW<[CortexA320WriteFDivDP], (instrs FDIVDrr)>;
+def : InstRW<[CortexA320WriteFDivHP], (instregex "^FDIVv.*16$")>;
+def : InstRW<[CortexA320WriteFDivSP], (instregex "^FDIVv.*32$")>;
+def : InstRW<[CortexA320WriteFDivDP], (instregex "^FDIVv.*64$")>;
+def : InstRW<[CortexA320WriteFSqrtHP], (instregex "^.*SQRT.*16$")>;
+def : InstRW<[CortexA320WriteFSqrtSP], (instregex "^.*SQRT.*32$")>;
+def : InstRW<[CortexA320WriteFSqrtDP], (instregex "^.*SQRT.*64$")>;
+
+def : InstRW<[CortexA320WriteFPALU_F3], (instrs FCSELHrrr, FCSELSrrr, FCSELDrrr)>;
+
+// Advanced SIMD integer instructions
+// ASIMD absolute diff
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]ABDv(2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]ABDv(16i8|4i32|8i16)")>;
+// ASIMD absolute diff accum
+def : InstRW<[CortexA320Write<6, CortexA320UnitVALU>], (instregex "[SU]ABAL?v")>;
+// ASIMD absolute diff long
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]ABDLv")>;
+// ASIMD arith #1
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "(ADD|SUB|NEG)v",
+  "[SU]R?HADDv", "[SU]HSUBv")>;
+// ASIMD arith #2
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$",
+  "[SU]ADDLPv(2i32_v1i64|4i16_v2i32|8i8_v4i16)$",
+  "ADDPv(2i32|4i16|8i8)$")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$",
+  "[SU]ADDLPv(16i8_v8i16|4i32_v2i64|8i16_v4i32)$",
+  "ADDPv(16i8|2i64|4i32|8i16)$")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$")>;
+// ASIMD arith #3
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex  "SADDLv", "UADDLv", "SADDWv",
+  "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex  "ADDHNv", "SUBHNv")>;
+// ASIMD arith #5
+def : InstRW<[CortexA320Write<8, CortexA320UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>;
+// ASIMD arith, reduce
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex  "ADDVv")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex  "SADDLVv", "UADDLVv")>;
+// ASIMD compare #1
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(2i64|4i32|8i16|16i8)")>;
+// ASIMD compare #2
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "CMTSTv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "CMTSTv(2i64|4i32|8i16|16i8)")>;
+// ASIMD logical $1
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v8i8",
+  "(ORR|BIC)v(2i32|4i16|8i8)$", "MVNIv(2i|2s|4i16)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v16i8",
+  "(ORR|BIC)v(16i8|4i32|8i16)$", "MVNIv(4i32|4s|8i16)")>;
+// ASIMD max/min, basic
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>;
+// SIMD max/min, reduce
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU](MAX|MIN)Vv")>;
+// ASIMD multiply, by element
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "MULv(2i32|4i16|4i32|8i16)_indexed$",
+  "SQR?DMULHv(1i16|1i32|2i32|4i16|4i32|8i16)_indexed$")>;
+// ASIMD multiply
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs PMULv8i8)>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs PMULv16i8)>;
+// ASIMD multiply accumulate
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "ML[AS]v(2i32|4i16|8i8)$")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "ML[AS]v(16i8|4i32|8i16)$")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "ML[AS]v(2i32|4i16|4i32|8i16)_indexed$")>;
+// ASIMD multiply accumulate half
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "SQRDML[AS]H[vi]")>;
+// ASIMD multiply accumulate long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]ML[AS]Lv")>;
+// ASIMD multiply accumulate long #2
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "SQDML[AS]L[iv]")>;
+// ASIMD dot product
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]DOTv8i8")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]DOTv16i8")>;
+// ASIMD dot product, by scalar
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]DOTlanev")>;
+// ASIMD multiply long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]MULLv", "SQDMULL[iv]")>;
+// ASIMD polynomial (8x8) multiply long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs PMULLv8i8, PMULLv16i8)>;
+// ASIMD pairwise add and accumulate
+def : InstRW<[CortexA320MCWrite<7, 2, CortexA320UnitVALU>], (instregex "[SU]ADALPv")>;
+// ASIMD shift accumulate
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>;
+// ASIMD shift accumulate #2
+def : InstRW<[CortexA320MCWrite<7, 2, CortexA320UnitVALU>], (instregex "[SU]RSRA[vd]")>;
+// ASIMD shift by immed
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "SHLd$", "SHLv",
+  "SLId$", "SRId$", "[SU]SHR[vd]", "SHRNv")>;
+// ASIMD shift by immed
+// SXTL and UXTL are aliases for SHLL
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[US]?SHLLv")>;
+// ASIMD shift by immed #2
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]RSHR(d|v2i32|v4i16|v8i8)",
+  "[SU]RSHRv(16i8|2i64|4i32|8i16)")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "RSHRNv(2i32|4i16|8i8)",
+  "RSHRNv(16i8|4i32|8i16)")>;
+// ASIMD shift by register
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]SHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]SHLv(2i64|4i32|8i16|16i8)")>;
+// ASIMD shift by register #2
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]RSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "[SU]RSHLv(2i64|4i32|8i16|16i8)")>;
+
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]QSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]QSHLv(2i64|4i32|8i16|16i8)")>;
+
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]QRSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "[SU]QRSHLv(2i64|4i32|8i16|16i8)")>;
+
+// Cryptography extensions
+// -----------------------------------------------------------------------------
+
+// Crypto AES ops
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^AES[DE]rr$", "^AESI?MCrr")>;
+
+// Crypto polynomial (64x64) multiply long
+def : InstRW<[CortexA320MCWrite<4, 0, CortexA320UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>;
+
+// Crypto SHA1 hash acceleration op
+// Crypto SHA1 schedule acceleration ops
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^SHA1(H|SU0|SU1)")>;
+
+// Crypto SHA1 hash acceleration ops
+// Crypto SHA256 hash acceleration ops
+def : InstRW<[CortexA320MCWrite<4, 0, CortexA320UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>;
+
+// Crypto SHA256 schedule acceleration ops
+def : InstRW<[CortexA320MCWrite<4, 0, CortexA320UnitVMC>], (instregex "^SHA256SU[01]")>;
+
+// Crypto SHA512 hash acceleration ops
+def : InstRW<[CortexA320MCWrite<9, 0, CortexA320UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>;
+
+// Crypto SHA3 ops
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instrs BCAX, EOR3)>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs XAR)>;
+def : InstRW<[CortexA320MCWrite<9, 0, CortexA320UnitVMC>], (instrs RAX1)>;
+
+
+// Crypto SM3 ops
+def : InstRW<[CortexA320MCWrite<9, 0, CortexA320UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$",
+                                                            "^SM3TT[12][AB]$")>;
+
+// Crypto SM4 ops
+def : InstRW<[CortexA320MCWrite<9, 0, CortexA320UnitVMC>], (instrs SM4E, SM4ENCKEY)>;
+
+// CRC
+// -----------------------------------------------------------------------------
+
+def : InstRW<[CortexA320MCWrite<2, 0, CortexA320UnitMAC>], (instregex "^CRC32")>;
+
+// SVE Predicate instructions
+
+// Loop control, based on predicate
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs BRKA_PPmP, BRKA_PPzP,
+                                                  BRKB_PPmP, BRKB_PPzP)>;
+
+// Loop control, based on predicate and flag setting
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs BRKAS_PPzP, BRKBS_PPzP)>;
+
+// Loop control, propagating
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>;
+
+// Loop control, propagating and flag setting
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs BRKNS_PPzP)>;
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>;
+
+
+// Loop control, based on GPR
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>],
+             (instregex "^WHILE(GE|GT|HI|HS|LE|LO|LS|LT)_P(WW|XX)_[BHSD]")>;
+
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]")>;
+
+// Loop terminate
+def : InstRW<[CortexA320Write<1, CortexA320UnitALU>], (instregex "^CTERM(EQ|NE)_(WW|XX)")>;
+
+// Predicate counting scalar
+def : InstRW<[CortexA320Write<1, CortexA320UnitALU>], (instrs ADDPL_XXI, ADDVL_XXI, RDVLI_XI)>;
+
+def : InstRW<[CortexA320Write<3, CortexA320UnitALU>],
+             (instregex "^CNT[BHWD]_XPiI")>;
+
+def : InstRW<[CortexA320Write<3, CortexA320UnitALU>],
+             (instregex "^(INC|DEC)[BHWD]_XPiI")>;
+
+def : InstRW<[CortexA320Write<5, CortexA320UnitALU>],
+             (instregex "^(SQINC|SQDEC|UQINC|UQDEC)[BHWD]_[XW]Pi(Wd)?I")>;
+
+// Predicate counting scalar, active predicate
+def : InstRW<[CortexA320Write<1, CortexA320UnitVALU>],
+             (instregex "^CNTP_XPP_[BHSD]")>;
+
+def : InstRW<[CortexA320Write<1, CortexA320UnitVALU>],
+             (instregex "^(DEC|INC)P_XP_[BHSD]")>;
+
+def : InstRW<[CortexA320Write<9, CortexA320UnitVALU>],
+             (instregex "^(SQDEC|SQINC|UQDEC|UQINC)P_XP_[BHSD]",
+                        "^(UQDEC|UQINC)P_WP_[BHSD]",
+                        "^(SQDEC|SQINC|UQDEC|UQINC)P_XPWd_[BHSD]")>;
+
+
+// Predicate counting vector, active predicate
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)P_ZP_[HSD]")>;
+
+// Predicate logical
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>],
+             (instregex "^(AND|BIC|EOR|NAND|NOR|ORN|ORR)_PPzPP")>;
+
+// Predicate logical, flag setting
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>],
+             (instregex "^(ANDS|BICS|EORS|NANDS|NORS|ORNS|ORRS)_PPzPP")>;
+
+// Predicate reverse
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^REV_PP_[BHSD]")>;
+
+// Predicate select
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs SEL_PPPP)>;
+
+// Predicate set
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^PFALSE", "^PTRUE_[BHSD]")>;
+
+// Predicate set/initialize, set flags
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^PTRUES_[BHSD]")>;
+
+// Predicate find first/next
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^PFIRST_B", "^PNEXT_[BHSD]")>;
+
+// Predicate test
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs PTEST_PP)>;
+
+// Predicate transpose
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^TRN[12]_PPP_[BHSDQ]")>;
+
+// Predicate unpack and widen
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instrs PUNPKHI_PP, PUNPKLO_PP)>;
+
+// Predicate zip/unzip
+def : InstRW<[CortexA320Write<2, CortexA320UnitVALU>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]")>;
+
+
+// SVE integer instructions
+// -----------------------------------------------------------------------------
+// Arithmetic, absolute diff
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^[SU]ABD_(ZPmZ|ZPZZ)_[BHSD]")>;
+
+// Arithmetic, absolute diff accum
+def : InstRW<[CortexA320MCWrite<6, 2, CortexA320UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]")>;
+
+// Arithmetic, absolute diff accum long
+def : InstRW<[CortexA320MCWrite<6, 2, CortexA320UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]")>;
+
+// Arithmetic, absolute diff long
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]")>;
+
+// Arithmetic, basic
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(ABS|CNOT|NEG)_ZPmZ_[BHSD]",
+                        "^(ADD|SUB|SUBR)_ZPmZ_[BHSD]",
+                        "^(ADD|SUB|SUBR)_ZPZZ_[BHSD]",
+                        "^(ADD|SUB)_ZZZ_[BHSD]",
+                        "^(ADD|SUB|SUBR)_ZI_[BHSD]",
+                        "^ADR_[SU]XTW_ZZZ_D_[0123]",
+                        "^ADR_LSL_ZZZ_[SD]_[0123]",
+                        "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]",
+                        "^SADDLBT_ZZZ_[HSD]",
+                        "^SSUBL(BT|TB)_ZZZ_[HSD]")>;
+
+// Arithmetic, complex
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^SQ(ABS|NEG)_ZPmZ_[BHSD]",
+                        "^SQ(ADD|SUB|SUBR)_ZPmZ_?[BHSD]",
+                        "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]",
+                        "^[SU]Q(ADD|SUB)_ZI_[BHSD]",
+                        "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]",
+                        "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]")>;
+def : InstRW<[CortexA320Write<8, CortexA320UnitVALU>],
+             (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]")>;
+
+// Arithmetic, large integer
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]")>;
+
+// Arithmetic, pairwise add
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^ADDP_ZPmZ_[BHSD]")>;
+
+// Arithmetic, pairwise add and accum long
+def : InstRW<[CortexA320MCWrite<7, 2, CortexA320UnitVALU>], (instregex "^[SU]ADALP_ZPmZ_[HSD]")>;
+
+// Arithmetic, shift
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]",
+                        "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]",
+                        "^(ASR|LSL|LSR)_ZPmI_[BHSD]",
+                        "^(ASR|LSL|LSR)_ZPZI_[BHSD]",
+                        "^(ASR|LSL|LSR)_ZPmZ_[BHSD]",
+                        "^(ASR|LSL|LSR)_ZPZZ_[BHSD]",
+                        "^(ASR|LSL|LSR)_ZZI_[BHSD]",
+                        "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]")>;
+// Arithmetic, shift right for divide
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^ASRD_ZPmI_[BHSD]",
+                        "^ASRD_ZPZI_[BHSD]")>;
+
+// Arithmetic, shift and accumulate
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^(SSRA|USRA)_ZZI_[BHSD]")>;
+
+def : InstRW<[CortexA320MCWrite<7, 2, CortexA320UnitVALU>],
+             (instregex "^(SRSRA|URSRA)_ZZI_[BHSD]")>;
+
+
+// Arithmetic, shift by immediate
+// Arithmetic, shift by immediate and insert
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(SHRNB|SHRNT|SSHLLB|SSHLLT|USHLLB|USHLLT|SLI|SRI)_ZZI_[BHSD]")>;
+
+// Arithmetic, shift complex
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]",
+                        "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_(ZPmZ|ZPZZ)_[BHSD]",
+                        "^(SQSHL|SQSHLU|UQSHL)_(ZPmI|ZPZI)_[BHSD]",
+                        "^SQSHRU?N[BT]_ZZI_[BHS]",
+                        "^UQR?SHRN[BT]_ZZI_[BHS]")>;
+
+// Arithmetic, shift rounding
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^(SRSHL|SRSHR|SRSHLR|URSHL|URSHLR|URSHR)_(ZPmZ|ZPZZ|ZPZI)_[BHSD]",
+                        "^[SU]RSHR_ZPmI_[BHSD]")>;
+
+// Bit manipulation
+def : InstRW<[CortexA320MCWrite<13, 12, CortexA320UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_B")>;
+
+def : InstRW<[CortexA320MCWrite<21, 20, CortexA320UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_H")>;
+
+def : InstRW<[CortexA320MCWrite<37, 36, CortexA320UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_S")>;
+
+def : InstRW<[CortexA320MCWrite<68, 67, CortexA320UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_D")>;
+
+
+// Bitwise select
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^(BSL|BSL1N|BSL2N|NBSL)_ZZZZ")>;
+
+// Count/reverse bits
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^(CLS|CLZ|RBIT)_ZPmZ_[BHSD]")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^CNT_ZPmZ_[BH]")>;
+def : InstRW<[CortexA320Write<8, CortexA320UnitVALU>], (instregex "^CNT_ZPmZ_S")>;
+def : InstRW<[CortexA320Write<12, CortexA320UnitVALU>], (instregex "^CNT_ZPmZ_D")>;
+// Broadcast logical bitmask immediate to vector
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs DUPM_ZI)>;
+
+// Compare and set flags
+def : InstRW<[CortexA320Write<5, CortexA320UnitVALU>],
+             (instregex "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_PPzZ[IZ]_[BHSD]",
+                        "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_WIDE_PPzZZ_[BHS]")>;
+
+// Complex add
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^CADD_ZZI_[BHSD]")>;
+
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^SQCADD_ZZI_[BHSD]")>;
+
+// Complex dot product 8-bit element
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs CDOT_ZZZ_S, CDOT_ZZZI_S)>;
+
+// Complex dot product 16-bit element
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs CDOT_ZZZ_D, CDOT_ZZZI_D)>;
+
+// Complex multiply-add B, H, S element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^CMLA_ZZZ_[BHS]",
+                                            "^CMLA_ZZZI_[HS]")>;
+
+// Complex multiply-add D element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs CMLA_ZZZ_D)>;
+
+// Conditional extract operations, scalar form
+def : InstRW<[CortexA320MCWrite<8, 2, CortexA320UnitVALU>], (instregex "^CLAST[AB]_RPZ_[BHSD]")>;
+
+// Conditional extract operations, SIMD&FP scalar and vector forms
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^CLAST[AB]_[VZ]PZ_[BHSD]",
+                                            "^COMPACT_ZPZ_[SD]",
+                                            "^SPLICE_ZPZZ?_[BHSD]")>;
+
+// Convert to floating point, 64b to float or convert to double
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_Dto[SD]")>;
+
+// Convert to floating point, 64b to half
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_DtoH")>;
+
+// Convert to floating point, 32b to single or half
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]")>;
+
+// Convert to floating point, 32b to double
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_StoD")>;
+
+// Convert to floating point, 16b to half
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_HtoH")>;
+
+// Copy, scalar
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],(instregex "^CPY_ZPmR_[BHSD]")>;
+
+// Copy, scalar SIMD&FP or imm
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^CPY_ZPm[IV]_[BHSD]",
+                                           "^CPY_ZPzI_[BHSD]")>;
+
+// Divides, 32 bit
+def : InstRW<[CortexA320MCWrite<15, 12, CortexA320UnitVMC>], (instregex "^[SU]DIVR?_(ZPmZ|ZPZZ)_S")>;
+
+// Divides, 64 bit
+def : InstRW<[CortexA320MCWrite<26, 23, CortexA320UnitVMC>], (instregex "^[SU]DIVR?_(ZPmZ|ZPZZ)_D")>;
+
+// Dot product, 8 bit
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^[SU]DOT_ZZZI?_S")>;
+
+// Dot product, 8 bit, using signed and unsigned integers
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs SUDOT_ZZZI, USDOT_ZZZI, USDOT_ZZZ)>;
+
+// Dot product, 16 bit
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^[SU]DOT_ZZZI?_D")>;
+
+// Duplicate, immediate and indexed form
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^DUP_ZI_[BHSD]",
+                                           "^DUP_ZZI_[BHSDQ]")>;
+
+// Duplicate, scalar form
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^DUP_ZR_[BHSD]")>;
+
+// Extend, sign or zero
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^[SU]XTB_ZPmZ_[HSD]",
+                                            "^[SU]XTH_ZPmZ_[SD]",
+                                            "^[SU]XTW_ZPmZ_[D]")>;
+
+// Extract
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_B)>;
+
+// Extract narrow saturating
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]",
+                                            "^SQXTUN[BT]_ZZ_[BHS]")>;
+
+// Extract/insert operation, SIMD and FP scalar form
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^LAST[AB]_VPZ_[BHSD]",
+                                            "^INSR_ZV_[BHSD]")>;
+
+// Extract/insert operation, scalar
+def : InstRW<[CortexA320MCWrite<8, 2, CortexA320UnitVALU>], (instregex "^LAST[AB]_RPZ_[BHSD]",
+                                                "^INSR_ZR_[BHSD]")>;
+
+// Histogram operations
+def : InstRW<[CortexA320MCWrite<8, 2, CortexA320UnitVALU>], (instregex "^HISTCNT_ZPzZZ_[SD]",
+                                                  "^HISTSEG_ZZZ")>;
+
+// Horizontal operations, B, H, S form, immediate operands only
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^INDEX_II_[BHS]")>;
+
+// Horizontal operations, B, H, S form, scalar, immediate operands/ scalar
+// operands only / immediate, scalar operands
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^INDEX_(IR|RI|RR)_[BHS]")>;
+
+// Horizontal operations, D form, immediate operands only
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs INDEX_II_D)>;
+
+// Horizontal operations, D form, scalar, immediate operands)/ scalar operands
+// only / immediate, scalar operands
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^INDEX_(IR|RI|RR)_D")>;
+
+// Logical
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(AND|EOR|ORR)_ZI",
+                        "^(AND|BIC|EOR|EOR|ORR)_ZZZ",
+                        "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]",
+                        "^(AND|BIC|EOR|NOT|ORR)_ZPZZ_[BHSD]")>;
+
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^EOR(BT|TB)_ZZZ_[BHSD]")>;
+
+// Max/min, basic and pairwise
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]",
+                                           "^[SU](MAX|MIN)P?_(ZPmZ|ZPZZ)_[BHSD]")>;
+
+// Matching operations
+def : InstRW<[CortexA320MCWrite<9, 2, CortexA320UnitVALU>], (instregex "^N?MATCH_PPzZZ_[BH]")>;
+
+// Matrix multiply-accumulate
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs SMMLA_ZZZ, UMMLA_ZZZ, USMMLA_ZZZ)>;
+
+// Move prefix
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]",
+                                           "^MOVPRFX_ZZ")>;
+
+// Multiply, B, H, S element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ|ZPZZ)_[BHS]",
+                                            "^[SU]MULH_(ZPmZ|ZZZ|ZPZZ)_[BHS]")>;
+
+// Multiply, D element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ|ZPZZ)_D",
+                                            "^[SU]MULH_(ZPmZ|ZZZ|ZPZZ)_D")>;
+
+// Multiply long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^[SU]MULL[BT]_ZZZI_[SD]",
+                                            "^[SU]MULL[BT]_ZZZ_[HSD]")>;
+
+// Multiply accumulate, B, H, S element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^ML[AS]_(ZZZI|ZPZZZ)_[BHS]",
+                                            "^(ML[AS]|MAD|MSB)_ZPmZZ_[BHS]")>;
+
+// Multiply accumulate, D element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^ML[AS]_(ZZZI|ZPZZZ)_D",
+                                            "^(ML[AS]|MAD|MSB)_ZPmZZ_D")>;
+
+// Multiply accumulate long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^[SU]ML[AS]L[BT]_ZZZ_[HSD]",
+                                            "^[SU]ML[AS]L[BT]_ZZZI_[SD]")>;
+
+// Multiply accumulate saturating doubling long regular
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQDML[AS](LB|LT|LBT)_ZZZ_[HSD]",
+                                            "^SQDML[AS](LB|LT)_ZZZI_[SD]")>;
+
+// Multiply saturating doubling high, B, H, S element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQDMULH_ZZZ_[BHS]",
+                                            "^SQDMULH_ZZZI_[HS]")>;
+
+// Multiply saturating doubling high, D element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instrs SQDMULH_ZZZ_D, SQDMULH_ZZZI_D)>;
+
+// Multiply saturating doubling long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQDMULL[BT]_ZZZ_[HSD]",
+                                            "^SQDMULL[BT]_ZZZI_[SD]")>;
+
+// Multiply saturating rounding doubling regular/complex accumulate, B, H, S
+// element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQRDML[AS]H_ZZZ_[BHS]",
+                                            "^SQRDCMLAH_ZZZ_[BHS]",
+                                            "^SQRDML[AS]H_ZZZI_[HS]",
+                                            "^SQRDCMLAH_ZZZI_[HS]")>;
+
+// Multiply saturating rounding doubling regular/complex accumulate, D element
+// size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQRDML[AS]H_ZZZI?_D",
+                                            "^SQRDCMLAH_ZZZ_D")>;
+
+// Multiply saturating rounding doubling regular/complex, B, H, S element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQRDMULH_ZZZ_[BHS]",
+                                            "^SQRDMULH_ZZZI_[HS]")>;
+
+// Multiply saturating rounding doubling regular/complex, D element size
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^SQRDMULH_ZZZI?_D")>;
+
+// Multiply/multiply long, (8x8) polynomial
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^PMUL_ZZZ_B")>;
+
+def : InstRW<[CortexA320Write<9, CortexA320UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]")>;
+
+
+// Predicate counting vector
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>],
+             (instregex "^(DEC|INC)[HWD]_ZPiI")>;
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^(SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI")>;
+
+// Reciprocal estimate
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^URECPE_ZPmZ_S", "^URSQRTE_ZPmZ_S")>;
+
+// Reduction, arithmetic, B form
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_B")>;
+
+// Reduction, arithmetic, H form
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_H")>;
+
+// Reduction, arithmetic, S form
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_S")>;
+
+// Reduction, arithmetic, D form
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_D")>;
+
+// Reduction, logical
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^(ANDV|EORV|ORV)_VPZ_[BHSD]")>;
+
+// Reverse, vector
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^REV_ZZ_[BHSD]",
+                                           "^REVB_ZPmZ_[HSD]",
+                                           "^REVH_ZPmZ_[SD]",
+                                           "^REVW_ZPmZ_D")>;
+
+// Select, vector form
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^SEL_ZPZZ_[BHSD]")>;
+
+// Table lookup
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^TBL_ZZZZ?_[BHSD]")>;
+
+// Table lookup extension
+def : InstRW<[CortexA320Write<8, CortexA320UnitVALU>], (instregex "^TBX_ZZZ_[BHSD]")>;
+
+// Transpose, vector form
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]")>;
+
+// Unpack and extend
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^[SU]UNPK(HI|LO)_ZZ_[HSD]")>;
+
+// Zip/unzip
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]")>;
+
+// SVE floating-point instructions
+// -----------------------------------------------------------------------------
+
+// Floating point absolute value/difference
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FAB[SD]_ZPmZ_[HSD]",
+                                                                  "^FAB[SD]_ZPZZ_[HSD]")>;
+
+// Floating point arithmetic
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ|ZPZI|ZPZZ)_[HSD]",
+                                           "^FADDP_ZPmZZ_[HSD]",
+                                           "^FNEG_ZPmZ_[HSD]",
+                                           "^FSUBR_(ZPm[IZ]|ZPZ[IZ])_[HSD]")>;
+
+// Floating point associative add, F16
+def : InstRW<[CortexA320MCWrite<32, 29, CortexA320UnitVALU>], (instrs FADDA_VPZ_H)>;
+
+// Floating point associative add, F32
+def : InstRW<[CortexA320MCWrite<16, 13, CortexA320UnitVALU>], (instrs FADDA_VPZ_S)>;
+
+// Floating point associative add, F64
+def : InstRW<[CortexA320MCWrite<8, 5, CortexA320UnitVALU>], (instrs FADDA_VPZ_D)>;
+
+// Floating point compare
+def : InstRW<[CortexA320Write<5, CortexA320UnitVALU>], (instregex "^FACG[ET]_PPzZZ_[HSD]",
+                                            "^FCM(EQ|GE|GT|NE)_PPzZ[0Z]_[HSD]",
+                                            "^FCM(LE|LT)_PPzZ0_[HSD]",
+                                            "^FCMUO_PPzZZ_[HSD]")>;
+
+// Floating point complex add
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCADD_ZPmZ_[HSD]")>;
+
+// Floating point complex multiply add
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FCMLA_ZPmZZ_[HSD]",
+                                           "^FCMLA_ZZZI_[HS]")>;
+
+// Floating point convert, long or narrow (F16 to F32 or F32 to F16)
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCVT_ZPmZ_(HtoS|StoH)",
+                                            "^FCVTLT_ZPmZ_HtoS",
+                                            "^FCVTNT_ZPmZ_StoH")>;
+
+// Floating point convert, long or narrow (F16 to F64, F32 to F64, F64 to F32
+// or F64 to F16)
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)",
+                                            "^FCVTLT_ZPmZ_StoD",
+                                            "^FCVTNT_ZPmZ_DtoS")>;
+
+// Floating point convert, round to odd
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCVTX_ZPmZ_DtoS", "FCVTXNT_ZPmZ_DtoS")>;
+
+// Floating point base2 log, F16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FLOGB_(ZPmZ|ZPZZ)_H")>;
+
+// Floating point base2 log, F32
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FLOGB_(ZPmZ|ZPZZ)_S")>;
+
+// Floating point base2 log, F64
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FLOGB_(ZPmZ|ZPZZ)_D")>;
+
+// Floating point convert to integer, F16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCVTZ[SU]_ZPmZ_HtoH")>;
+
+// Floating point convert to integer, F32
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)")>;
+
+// Floating point convert to integer, F64
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)")>;
+
+// Floating point copy
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^FCPY_ZPmI_[HSD]",
+                                           "^FDUP_ZI_[HSD]")>;
+
+// Floating point divide, F16
+def : InstRW<[CortexA320MCWrite<8, 5, CortexA320UnitVMC>], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_H")>;
+
+// Floating point divide, F32
+def : InstRW<[CortexA320MCWrite<13, 10, CortexA320UnitVMC>], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_S")>;
+
+// Floating point divide, F64
+def : InstRW<[CortexA320MCWrite<22, 19, CortexA320UnitVMC>], (instregex "^FDIVR?_(ZPmZ|ZPZZ)_D")>;
+
+// Floating point min/max pairwise
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]")>;
+
+// Floating point min/max
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^F(MAX|MIN)(NM)?_(ZPm[IZ]|ZPZZ|ZPZI)_[HSD]")>;
+
+// Floating point multiply
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^(FSCALE|FMULX)_(ZPmZ|ZPZZ)_[HSD]",
+                                           "^FMUL_(ZPm[IZ]|ZZZI?|ZPZI|ZPZZ)_[HSD]")>;
+
+// Floating point multiply accumulate
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>],
+             (instregex "^FML[AS]_(ZPmZZ|ZZZI|ZPZZZ)_[HSD]",
+                        "^(FMAD|FNMAD|FNML[AS]|FN?MSB)_(ZPmZZ|ZPZZZ)_[HSD]")>;
+
+// Floating point multiply add/sub accumulate long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FML[AS]L[BT]_ZZZI?_SHH")>;
+
+// Floating point reciprocal estimate, F16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FRECPE_ZZ_H", "^FRECPX_ZPmZ_H",
+                                         "^FRSQRTE_ZZ_H")>;
+
+// Floating point reciprocal estimate, F32
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FRECPE_ZZ_S", "^FRECPX_ZPmZ_S",
+                                         "^FRSQRTE_ZZ_S")>;
+// Floating point reciprocal estimate, F64
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>],(instregex "^FRECPE_ZZ_D", "^FRECPX_ZPmZ_D",
+                                         "^FRSQRTE_ZZ_D")>;
+
+// Floating point reciprocal step
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^F(RECPS|RSQRTS)_ZZZ_[HSD]")>;
+
+// Floating point reduction, F16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^(FMAXNMV|FMAXV|FMINNMV|FMINV)_VPZ_[HSD]")>;
+
+// Floating point reduction, F32
+def : InstRW<[CortexA320MCWrite<12, 11, CortexA320UnitVALU>],
+             (instregex "^FADDV_VPZ_H")>;
+
+def : InstRW<[CortexA320MCWrite<8, 5, CortexA320UnitVALU>],
+             (instregex "^FADDV_VPZ_S")>;
+
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>],
+             (instregex "^FADDV_VPZ_D")>;
+
+
+// Floating point round to integral, F16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H")>;
+
+// Floating point round to integral, F32
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S")>;
+
+// Floating point round to integral, F64
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D")>;
+
+// Floating point square root, F16
+def : InstRW<[CortexA320MCWrite<11, 5, CortexA320UnitVMC>], (instregex "^FSQRT_ZPmZ_H")>;
+
+// Floating point square root, F32
+def : InstRW<[CortexA320MCWrite<14, 9, CortexA320UnitVMC>], (instregex "^FSQRT_ZPmZ_S")>;
+
+// Floating point square root, F64
+def : InstRW<[CortexA320MCWrite<25, 19, CortexA320UnitVMC>], (instregex "^FSQRT_ZPmZ_D")>;
+
+// Floating point trigonometric exponentiation
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FEXPA_ZZ_[HSD]")>;
+
+// Floating point trigonometric multiply add
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FTMAD_ZZI_[HSD]")>;
+
+// Floating point trigonometric, miscellaneous
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^FTSMUL_ZZZ_[HSD]")>;
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]")>;
+
+
+// SVE BFloat16 (BF16) instructions
+// -----------------------------------------------------------------------------
+
+// Convert, F32 to BF16
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instrs BFCVT_ZPmZ, BFCVTNT_ZPmZ)>;
+
+// Dot product
+def : InstRW<[CortexA320Write_11cyc_1VMAC_1VALU], (instrs BFDOT_ZZI, BFDOT_ZZZ)>;
+
+// Matrix multiply accumulate
+def : InstRW<[CortexA320Write_16cyc_1VMAC_1VALU], (instrs BFMMLA_ZZZ)>;
+
+// Multiply accumulate long
+def : InstRW<[CortexA320Write<4, CortexA320UnitVMAC>], (instregex "^BFMLAL[BT]_ZZZ(I)?")>;
+
+// SVE Load instructions
+// -----------------------------------------------------------------------------
+
+// Load vector
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instrs LDR_ZXI)>;
+
+// Load predicate
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instrs LDR_PXI)>;
+
+// Contiguous load, scalar + imm
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LD1[BHWD]_IMM$",
+                                           "^LD1S?B_[HSD]_IMM$",
+                                           "^LD1S?H_[SD]_IMM$",
+                                           "^LD1S?W_D_IMM$" )>;
+// Contiguous load, scalar + scalar
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LD1[BHWD]$",
+                                             "^LD1S?B_[HSD]$",
+                                             "^LD1S?H_[SD]$",
+                                             "^LD1S?W_D$" )>;
+
+// Contiguous load broadcast, scalar + imm
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LD1R[BHWD]_IMM$",
+                                           "^LD1RSW_IMM$",
+                                           "^LD1RS?B_[HSD]_IMM$",
+                                           "^LD1RS?H_[SD]_IMM$",
+                                           "^LD1RS?W_D_IMM$",
+                                           "^LD1RQ_[BHWD]_IMM$")>;
+
+// Contiguous load broadcast, scalar + scalar
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LD1RQ_[BHWD]$")>;
+
+// Non temporal load, scalar + imm
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRI$")>;
+
+// Non temporal load, scalar + scalar
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRR$")>;
+
+// Non temporal gather load, vector + scalar 32-bit element size
+def : InstRW<[CortexA320MCWrite<9, 9, CortexA320UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S$",
+                                              "^LDNT1S[BH]_ZZR_S$")>;
+
+// Non temporal gather load, vector + scalar 64-bit element size
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D$")>;
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>], (instrs LDNT1D_ZZR_D)>;
+
+// Contiguous first faulting load, scalar + scalar
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LDFF1[BHWD]$",
+                                              "^LDFF1S?B_[HSD]$",
+                                              "^LDFF1S?H_[SD]$",
+                                              "^LDFF1S?W_D$")>;
+
+// Contiguous non faulting load, scalar + imm
+def : InstRW<[CortexA320Write<3, CortexA320UnitLdSt>], (instregex "^LDNF1[BHWD]_IMM$",
+                                           "^LDNF1S?B_[HSD]_IMM$",
+                                           "^LDNF1S?H_[SD]_IMM$",
+                                           "^LDNF1S?W_D_IMM$")>;
+
+// Contiguous Load two structures to two vectors, scalar + imm
+def : InstRW<[CortexA320MCWrite<3, 1, CortexA320UnitLdSt>], (instregex "^LD2[BHWD]_IMM$")>;
+
+// Contiguous Load two structures to two vectors, scalar + scalar
+def : InstRW<[CortexA320MCWrite<3, 2, CortexA320UnitLdSt>], (instregex "^LD2[BHWD]$")>;
+
+// Contiguous Load three structures to three vectors, scalar + imm
+def : InstRW<[CortexA320MCWrite<5, 3, CortexA320UnitLdSt>], (instregex "^LD3[BHWD]_IMM$")>;
+
+// Contiguous Load three structures to three vectors, scalar + scalar
+def : InstRW<[CortexA320MCWrite<5, 3, CortexA320UnitLdSt>], (instregex "^LD3[BHWD]$")>;
+
+// Contiguous Load four structures to four vectors, scalar + imm
+def : InstRW<[CortexA320MCWrite<5, 3, CortexA320UnitLdSt>], (instregex "^LD4[BHWD]_IMM$")>;
+
+// Contiguous Load four structures to four vectors, scalar + scalar
+def : InstRW<[CortexA320MCWrite<5, 3, CortexA320UnitLdSt>], (instregex "^LD4[BHWD]$")>;
+
+// Gather load, vector + imm, 32-bit element size
+def : InstRW<[CortexA320MCWrite<9, 9, CortexA320UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM$",
+                                              "^GLD(FF)?1W_IMM$")>;
+
+// Gather load, vector + imm, 64-bit element size
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM$",
+                                              "^GLD(FF)?1D_IMM$")>;
+
+// Gather load, 64-bit element size
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>],
+             (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW(_SCALED)?$",
+                        "^GLD(FF)?1S?[BHW]_D(_SCALED)?$",
+                        "^GLD(FF)?1D_[SU]XTW(_SCALED)?$",
+                        "^GLD(FF)?1D(_SCALED)?$")>;
+
+// Gather load, 32-bit scaled offset
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>],
+             (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED$",
+                        "^GLD(FF)?1W_[SU]XTW_SCALED")>;
+
+// Gather load, 32-bit unpacked unscaled offset
+def : InstRW<[CortexA320MCWrite<7, 7, CortexA320UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW$",
+                                              "^GLD(FF)?1W_[SU]XTW$")>;
+
+def : InstRW<[CortexA320Write<0, CortexA320UnitVALU>], (instregex "^PRF(B|H|W|D).*")>;
+// SVE Store instructions
+// -----------------------------------------------------------------------------
+
+// Store from predicate reg
+def : InstRW<[CortexA320VSt0], (instrs STR_PXI)>;
+
+// Store from vector reg
+def : InstRW<[CortexA320VSt0], (instrs STR_ZXI)>;
+
+// Contiguous store, scalar + imm
+def : InstRW<[CortexA320VSt0], (instregex "^ST1[BHWD]_IMM$",
+                                                "^ST1B_[HSD]_IMM$",
+                                                "^ST1H_[SD]_IMM$",
+                                                "^ST1W_D_IMM$")>;
+
+// Contiguous store, scalar + scalar
+def : InstRW<[CortexA320VSt0], (instregex "^ST1H(_[SD])?$")>;
+def : InstRW<[CortexA320VSt0], (instregex "^ST1[BWD]$",
+                                                "^ST1B_[HSD]$",
+                                                "^ST1W_D$")>;
+
+// Contiguous store two structures from two vectors, scalar + imm
+def : InstRW<[CortexA320VSt<11>], (instregex "^ST2[BHWD]_IMM$")>;
+
+// Contiguous store two structures from two vectors, scalar + scalar
+def : InstRW<[CortexA320VSt<11>], (instrs ST2H)>;
+
+// Contiguous store two structures from two vectors, scalar + scalar
+def : InstRW<[CortexA320VSt<11>], (instregex "^ST2[BWD]$")>;
+
+// Contiguous store three structures from three vectors, scalar + imm
+def : InstRW<[CortexA320VSt<25>], (instregex "^ST3[BHW]_IMM$")>;
+def : InstRW<[CortexA320VSt<14>], (instregex "^ST3D_IMM$")>;
+
+// Contiguous store three structures from three vectors, scalar + scalar
+def : InstRW<[CortexA320VSt<25>], (instregex "^ST3[BHW]$")>;
+def : InstRW<[CortexA320VSt<14>], (instregex "^ST3D$")>;
+
+// Contiguous store four structures from four vectors, scalar + imm
+def : InstRW<[CortexA320VSt<50>], (instregex "^ST4[BHW]_IMM$")>;
+def : InstRW<[CortexA320VSt<25>], (instregex "^ST4D_IMM$")>;
+
+// Contiguous store four structures from four vectors, scalar + scalar
+def : InstRW<[CortexA320VSt<50>], (instregex "^ST4[BHW]$")>;
+
+// Contiguous store four structures from four vectors, scalar + scalar
+def : InstRW<[CortexA320VSt<25>], (instregex "^ST4D$")>;
+
+// Non temporal store, scalar + imm
+def : InstRW<[CortexA320VSt0], (instregex "^STNT1[BHWD]_ZRI$")>;
+
+// Non temporal store, scalar + scalar
+def : InstRW<[CortexA320VSt0], (instrs STNT1H_ZRR)>;
+def : InstRW<[CortexA320VSt0], (instregex "^STNT1[BWD]_ZRR$")>;
+
+// Scatter non temporal store, vector + scalar 32-bit element size
+def : InstRW<[CortexA320VSt<9>], (instregex "^STNT1[BHW]_ZZR_S")>;
+
+// Scatter non temporal store, vector + scalar 64-bit element size
+def : InstRW<[CortexA320VSt<7>], (instregex "^STNT1[BHWD]_ZZR_D")>;
+
+// Scatter store vector + imm 32-bit element size
+def : InstRW<[CortexA320VSt<9>], (instregex "^SST1[BH]_S_IMM$",
+                                                "^SST1W_IMM$")>;
+
+// Scatter store vector + imm 64-bit element size
+def : InstRW<[CortexA320VSt<7>], (instregex "^SST1[BHW]_D_IMM$",
+                                                "^SST1D_IMM$")>;
+
+// Scatter store, 32-bit scaled offset
+def : InstRW<[CortexA320VSt<8>],
+             (instregex "^SST1(H_S|W)_[SU]XTW_SCALED$")>;
+
+// Scatter store, 32-bit unpacked unscaled offset
+def : InstRW<[CortexA320VSt<8>], (instregex "^SST1[BHW]_D_[SU]XTW$",
+                                                "^SST1D_[SU]XTW$")>;
+
+// Scatter store, 32-bit unpacked scaled offset
+def : InstRW<[CortexA320VSt<8>], (instregex "^SST1[HW]_D_[SU]XTW_SCALED$",
+                                                "^SST1D_[SU]XTW_SCALED$")>;
+
+// Scatter store, 32-bit unscaled offset
+def : InstRW<[CortexA320VSt<8>], (instregex "^SST1[BH]_S_[SU]XTW$",
+                                                "^SST1W_[SU]XTW$")>;
+
+// Scatter store, 64-bit scaled offset
+def : InstRW<[CortexA320VSt<8>], (instregex "^SST1[HW]_D_SCALED$",
+                                                "^SST1D_SCALED$")>;
+
+// Scatter store, 64-bit unscaled offset
+def : InstRW<[CortexA320VSt<8>], (instregex "^SST1[BHW]_D$",
+                                                "^SST1D$")>;
+
+// SVE Miscellaneous instructions
+// -----------------------------------------------------------------------------
+
+// Read first fault register, unpredicated
+def : InstRW<[CortexA320Write<1, CortexA320UnitALU>], (instrs RDFFR_P)>;
+
+// Read first fault register, predicated
+def : InstRW<[CortexA320Write<3, CortexA320UnitALU>], (instrs RDFFR_PPz)>;
+
+// Read first fault register and set flags
+def : InstRW<[CortexA320Write<3, CortexA320UnitALU>], (instrs RDFFRS_PPz)>;
+
+// Set first fault register
+// Write to first fault register
+def : InstRW<[CortexA320Write<1, CortexA320UnitALU>], (instrs SETFFR, WRFFR)>;
+
+// SVE Cryptographic instructions
+// -----------------------------------------------------------------------------
+
+// Crypto AES ops
+def : InstRW<[CortexA320Write<3, CortexA320UnitVALU>], (instregex "^AES[DE]_ZZZ_B$",
+                                           "^AESI?MC_ZZ_B$")>;
+
+// Crypto SHA3 ops
+def : InstRW<[CortexA320Write<4, CortexA320UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$",
+                                            "^XAR_ZZZI_[BHSD]$")>;
+
+def : InstRW<[CortexA320MC_RC0Write<9, CortexA320UnitVMC>], (instregex "^RAX1_ZZZ_D$")>;
+
+// Crypto SM4 ops
+def : InstRW<[CortexA320MC_RC0Write<9, CortexA320UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>;
+
+}
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-basic-instructions.s
new file mode 100644
index 000000000000..35b5d5b2ce43
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-basic-instructions.s
@@ -0,0 +1,3721 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a320 -instruction-tables < %s | FileCheck %s
+
+#------------------------------------------------------------------------------
+# Add/sub (immediate)
+#------------------------------------------------------------------------------
+
+add      w2, w3, #4095
+add      w30, w29, #1, lsl #12
+add      w13, w5, #4095, lsl #12
+add      x5, x7, #1638
+add      w20, wsp, #801
+add      wsp, wsp, #1104
+add      wsp, w30, #4084
+add      x0, x24, #291
+add      x3, x24, #4095, lsl #12
+add      x8, sp, #1074
+add      sp, x29, #3816
+sub      w0, wsp, #4077
+sub      w4, w20, #546, lsl #12
+sub      sp, sp, #288
+sub      wsp, w19, #16
+adds     w13, w23, #291, lsl #12
+cmn      w2, #4095
+adds     w20, wsp, #0
+cmn      x3, #1, lsl #12
+cmp      sp, #20, lsl #12
+cmp      x30, #4095
+subs     x4, sp, #3822
+cmn      w3, #291, lsl #12
+cmn      wsp, #1365
+cmn      sp, #1092, lsl #12
+mov      sp, x30
+mov      wsp, w20
+mov      x11, sp
+mov      w24, wsp
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+add      w3, w5, w7
+add      wzr, w3, w5
+add      w20, wzr, w4
+add      w4, w6, wzr
+add      w11, w13, w15
+add      w9, w3, wzr, lsl #10
+add      w17, w29, w20, lsl #31
+add      w21, w22, w23, lsr #0
+add      w24, w25, w26, lsr #18
+add      w27, w28, w29, lsr #31
+add      w2, w3, w4, asr #0
+add      w5, w6, w7, asr #21
+add      w8, w9, w10, asr #31
+add      x3, x5, x7
+add      xzr, x3, x5
+add      x20, xzr, x4
+add      x4, x6, xzr
+add      x11, x13, x15
+add      x9, x3, xzr, lsl #10
+add      x17, x29, x20, lsl #63
+add      x21, x22, x23, lsr #0
+add      x24, x25, x26, lsr #18
+add      x27, x28, x29, lsr #63
+add      x2, x3, x4, asr #0
+add      x5, x6, x7, asr #21
+add      x8, x9, x10, asr #63
+adds     w3, w5, w7
+cmn      w3, w5
+adds     w20, wzr, w4
+adds     w4, w6, wzr
+adds     w11, w13, w15
+adds     w9, w3, wzr, lsl #10
+adds     w17, w29, w20, lsl #31
+adds     w21, w22, w23, lsr #0
+adds     w24, w25, w26, lsr #18
+adds     w27, w28, w29, lsr #31
+adds     w2, w3, w4, asr #0
+adds     w5, w6, w7, asr #21
+adds     w8, w9, w10, asr #31
+adds     x3, x5, x7
+cmn      x3, x5
+adds     x20, xzr, x4
+adds     x4, x6, xzr
+adds     x11, x13, x15
+adds     x9, x3, xzr, lsl #10
+adds     x17, x29, x20, lsl #63
+adds     x21, x22, x23, lsr #0
+adds     x24, x25, x26, lsr #18
+adds     x27, x28, x29, lsr #63
+adds     x2, x3, x4, asr #0
+adds     x5, x6, x7, asr #21
+adds     x8, x9, x10, asr #63
+sub      w3, w5, w7
+sub      wzr, w3, w5
+sub      w4, w6, wzr
+sub      w11, w13, w15
+sub      w9, w3, wzr, lsl #10
+sub      w17, w29, w20, lsl #31
+sub      w21, w22, w23, lsr #0
+sub      w24, w25, w26, lsr #18
+sub      w27, w28, w29, lsr #31
+sub      w2, w3, w4, asr #0
+sub      w5, w6, w7, asr #21
+sub      w8, w9, w10, asr #31
+sub      x3, x5, x7
+sub      xzr, x3, x5
+sub      x4, x6, xzr
+sub      x11, x13, x15
+sub      x9, x3, xzr, lsl #10
+sub      x17, x29, x20, lsl #63
+sub      x21, x22, x23, lsr #0
+sub      x24, x25, x26, lsr #18
+sub      x27, x28, x29, lsr #63
+sub      x2, x3, x4, asr #0
+sub      x5, x6, x7, asr #21
+sub      x8, x9, x10, asr #63
+subs     w3, w5, w7
+cmp      w3, w5
+subs     w4, w6, wzr
+subs     w11, w13, w15
+subs     w9, w3, wzr, lsl #10
+subs     w17, w29, w20, lsl #31
+subs     w21, w22, w23, lsr #0
+subs     w24, w25, w26, lsr #18
+subs     w27, w28, w29, lsr #31
+subs     w2, w3, w4, asr #0
+subs     w5, w6, w7, asr #21
+subs     w8, w9, w10, asr #31
+subs     x3, x5, x7
+cmp      x3, x5
+subs     x4, x6, xzr
+subs     x11, x13, x15
+subs     x9, x3, xzr, lsl #10
+subs     x17, x29, x20, lsl #63
+subs     x21, x22, x23, lsr #0
+subs     x24, x25, x26, lsr #18
+subs     x27, x28, x29, lsr #63
+subs     x2, x3, x4, asr #0
+subs     x5, x6, x7, asr #21
+subs     x8, x9, x10, asr #63
+cmn      wzr, w4
+cmn      w5, wzr
+cmn      w6, w7
+cmn      w8, w9, lsl #15
+cmn      w10, w11, lsl #31
+cmn      w12, w13, lsr #0
+cmn      w14, w15, lsr #21
+cmn      w16, w17, lsr #31
+cmn      w18, w19, asr #0
+cmn      w20, w21, asr #22
+cmn      w22, w23, asr #31
+cmn      x0, x3
+cmn      xzr, x4
+cmn      x5, xzr
+cmn      x6, x7
+cmn      x8, x9, lsl #15
+cmn      x10, x11, lsl #63
+cmn      x12, x13, lsr #0
+cmn      x14, x15, lsr #41
+cmn      x16, x17, lsr #63
+cmn      x18, x19, asr #0
+cmn      x20, x21, asr #55
+cmn      x22, x23, asr #63
+cmp      w0, w3
+cmp      wzr, w4
+cmp      w5, wzr
+cmp      w6, w7
+cmp      w8, w9, lsl #15
+cmp      w10, w11, lsl #31
+cmp      w12, w13, lsr #0
+cmp      w14, w15, lsr #21
+cmp      w18, w19, asr #0
+cmp      w20, w21, asr #22
+cmp      w22, w23, asr #31
+cmp      x0, x3
+cmp      xzr, x4
+cmp      x5, xzr
+cmp      x6, x7
+cmp      x8, x9, lsl #15
+cmp      x10, x11, lsl #63
+cmp      x12, x13, lsr #0
+cmp      x14, x15, lsr #41
+cmp      x16, x17, lsr #63
+cmp      x18, x19, asr #0
+cmp      x20, x21, asr #55
+cmp      x22, x23, asr #63
+cmp      wzr, w0
+cmp      xzr, x0
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+adc      w29, w27, w25
+adc      wzr, w3, w4
+adc      w9, wzr, w10
+adc      w20, w0, wzr
+adc      x29, x27, x25
+adc      xzr, x3, x4
+adc      x9, xzr, x10
+adc      x20, x0, xzr
+adcs     w29, w27, w25
+adcs     wzr, w3, w4
+adcs     w9, wzr, w10
+adcs     w20, w0, wzr
+adcs     x29, x27, x25
+adcs     xzr, x3, x4
+adcs     x9, xzr, x10
+adcs     x20, x0, xzr
+sbc      w29, w27, w25
+sbc      wzr, w3, w4
+ngc      w9, w10
+sbc      w20, w0, wzr
+sbc      x29, x27, x25
+sbc      xzr, x3, x4
+ngc      x9, x10
+sbc      x20, x0, xzr
+sbcs     w29, w27, w25
+sbcs     wzr, w3, w4
+ngcs     w9, w10
+sbcs     w20, w0, wzr
+sbcs     x29, x27, x25
+sbcs     xzr, x3, x4
+ngcs     x9, x10
+sbcs     x20, x0, xzr
+ngc      w3, w12
+ngc      wzr, w9
+ngc      w23, wzr
+ngc      x29, x30
+ngc      xzr, x0
+ngc      x0, xzr
+ngcs     w3, w12
+ngcs     wzr, w9
+ngcs     w23, wzr
+ngcs     x29, x30
+ngcs     xzr, x0
+ngcs     x0, xzr
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+sbfx     x1, x2, #3, #2
+asr      x3, x4, #63
+asr      wzr, wzr, #31
+sbfx     w12, w9, #0, #1
+ubfiz    x4, x5, #52, #11
+ubfx     xzr, x4, #0, #1
+ubfiz    x4, xzr, #1, #6
+lsr      x5, x6, #12
+bfi      x4, x5, #52, #11
+bfxil    xzr, x4, #0, #1
+bfi      x4, xzr, #1, #6
+bfxil    x5, x6, #12, #52
+sxtb     w1, w2
+sxtb     xzr, w3
+sxth     w9, w10
+sxth     x0, w1
+sxtw     x3, w30
+uxtb     w1, w2
+uxth     w9, w10
+ubfx     x3, x30, #0, #32
+asr      w3, w2, #0
+asr      w9, w10, #31
+asr      x20, x21, #63
+asr      w1, wzr, #3
+lsr      w3, w2, #0
+lsr      w9, w10, #31
+lsr      x20, x21, #63
+lsr      wzr, wzr, #3
+lsr      w3, w2, #0
+lsl      w9, w10, #31
+lsl      x20, x21, #63
+lsl      w1, wzr, #3
+sbfx     w9, w10, #0, #1
+sbfiz    x2, x3, #63, #1
+asr      x19, x20, #0
+sbfiz    x9, x10, #5, #59
+asr      w9, w10, #0
+sbfiz    w11, w12, #31, #1
+sbfiz    w13, w14, #29, #3
+sbfiz    xzr, xzr, #10, #11
+sbfx     w9, w10, #0, #1
+asr      x2, x3, #63
+asr      x19, x20, #0
+asr      x9, x10, #5
+asr      w9, w10, #0
+asr      w11, w12, #31
+asr      w13, w14, #29
+sbfx     xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfi      x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfi      x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfi      w11, w12, #31, #1
+bfi      w13, w14, #29, #3
+bfi      xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfxil    x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfxil    x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfxil    w11, w12, #31, #1
+bfxil    w13, w14, #29, #3
+bfxil    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsl      x2, x3, #63
+lsr      x19, x20, #0
+lsl      x9, x10, #5
+lsr      w9, w10, #0
+lsl      w11, w12, #31
+lsl      w13, w14, #29
+ubfiz    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsr      x2, x3, #63
+lsr      x19, x20, #0
+lsr      x9, x10, #5
+lsr      w9, w10, #0
+lsr      w11, w12, #31
+lsr      w13, w14, #29
+ubfx     xzr, xzr, #10, #11
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+cbz      w5, #4
+cbz      x5, #0
+cbnz     x2, #-4
+cbnz     x26, #1048572
+cbz      wzr, #0
+cbnz     xzr, #0
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b.ne #4
+b.ge #1048572
+b.ge #-4
+
+#------------------------------------------------------------------------------
+# Conditional compare (immediate)
+#------------------------------------------------------------------------------
+
+ccmp w1, #31, #0, eq
+ccmp w3, #0, #15, hs
+ccmp wzr, #15, #13, hs
+ccmp x9, #31, #0, le
+ccmp x3, #0, #15, gt
+ccmp xzr, #5, #7, ne
+ccmn w1, #31, #0, eq
+ccmn w3, #0, #15, hs
+ccmn wzr, #15, #13, hs
+ccmn x9, #31, #0, le
+ccmn x3, #0, #15, gt
+ccmn xzr, #5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional compare (register)
+#------------------------------------------------------------------------------
+
+ccmp w1, wzr, #0, eq
+ccmp w3, w0, #15, hs
+ccmp wzr, w15, #13, hs
+ccmp x9, xzr, #0, le
+ccmp x3, x0, #15, gt
+ccmp xzr, x5, #7, ne
+ccmn w1, wzr, #0, eq
+ccmn w3, w0, #15, hs
+ccmn wzr, w15, #13, hs
+ccmn x9, xzr, #0, le
+ccmn x3, x0, #15, gt
+ccmn xzr, x5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+csel     w1, w0, w19, ne
+csel     wzr, w5, w9, eq
+csel     w9, wzr, w30, gt
+csel     w1, w28, wzr, mi
+csel     x19, x23, x29, lt
+csel     xzr, x3, x4, ge
+csel     x5, xzr, x6, hs
+csel     x7, x8, xzr, lo
+csinc    w1, w0, w19, ne
+csinc    wzr, w5, w9, eq
+csinc    w9, wzr, w30, gt
+csinc    w1, w28, wzr, mi
+csinc    x19, x23, x29, lt
+csinc    xzr, x3, x4, ge
+csinc    x5, xzr, x6, hs
+csinc    x7, x8, xzr, lo
+csinv    w1, w0, w19, ne
+csinv    wzr, w5, w9, eq
+csinv    w9, wzr, w30, gt
+csinv    w1, w28, wzr, mi
+csinv    x19, x23, x29, lt
+csinv    xzr, x3, x4, ge
+csinv    x5, xzr, x6, hs
+csinv    x7, x8, xzr, lo
+csneg    w1, w0, w19, ne
+csneg    wzr, w5, w9, eq
+csneg    w9, wzr, w30, gt
+csneg    w1, w28, wzr, mi
+csneg    x19, x23, x29, lt
+csneg    xzr, x3, x4, ge
+csneg    x5, xzr, x6, hs
+csneg    x7, x8, xzr, lo
+cset    w3, eq
+cset    x9, pl
+csetm    w20, ne
+csetm    x30, ge
+csinc    w2, wzr, wzr, al
+csinv    x3, xzr, xzr, nv
+cinc    w3, w5, gt
+cinc    wzr, w4, le
+cset    w9, lt
+cinc    x3, x5, gt
+cinc    xzr, x4, le
+cset    x9, lt
+csinc   w5, w6, w6, nv
+csinc   x1, x2, x2, al
+cinv    w3, w5, gt
+cinv    wzr, w4, le
+csetm   w9, lt
+cinv    x3, x5, gt
+cinv    xzr, x4, le
+csetm   x9, lt
+csinv   x1, x0, x0, al
+csinv   w9, w8, w8, nv
+cneg     w3, w5, gt
+cneg     wzr, w4, le
+cneg     w9, wzr, lt
+cneg     x3, x5, gt
+cneg     xzr, x4, le
+cneg     x9, xzr, lt
+csneg    x4, x8, x8, al
+csinv    w9, w8, w8, nv
+
+#------------------------------------------------------------------------------
+# Data-processing (1 source)
+#------------------------------------------------------------------------------
+
+rbit	w0, w7
+rbit   x18, x3
+rev16	w17, w1
+rev16	x5, x2
+rev	w18, w0
+rev32	x20, x1
+rev	x22, x2
+clz	w24, w3
+clz	x26, x4
+cls	w3, w5
+cls	x20, x5
+
+#------------------------------------------------------------------------------
+# Data-processing (2 source)
+#------------------------------------------------------------------------------
+
+udiv	w0, w7, w10
+udiv	x9, x22, x4
+sdiv	w12, w21, w0
+sdiv	x13, x2, x1
+lsl	w11, w12, w13
+lsl	x14, x15, x16
+lsr	w17, w18, w19
+lsr	x20, x21, x22
+asr	w23, w24, w25
+asr	x26, x27, x28
+ror	w0, w1, w2
+ror    x3, x4, x5
+lsl	w6, w7, w8
+lsl	x9, x10, x11
+lsr	w12, w13, w14
+lsr	x15, x16, x17
+asr	w18, w19, w20
+asr	x21, x22, x23
+ror	w24, w25, w26
+ror	x27, x28, x29
+
+#------------------------------------------------------------------------------
+# Data-processing (3 sources)
+#------------------------------------------------------------------------------
+
+smulh    x30, x29, x28
+smulh    xzr, x27, x26
+umulh    x30, x29, x28
+umulh    x23, x30, xzr
+madd     w1, w3, w7, w4
+madd     wzr, w0, w9, w11
+madd     w13, wzr, w4, w4
+madd     w19, w30, wzr, w29
+mul      w4, w5, w6
+madd     x1, x3, x7, x4
+madd     xzr, x0, x9, x11
+madd     x13, xzr, x4, x4
+madd     x19, x30, xzr, x29
+mul      x4, x5, x6
+msub     w1, w3, w7, w4
+msub     wzr, w0, w9, w11
+msub     w13, wzr, w4, w4
+msub     w19, w30, wzr, w29
+mneg     w4, w5, w6
+msub     x1, x3, x7, x4
+msub     xzr, x0, x9, x11
+msub     x13, xzr, x4, x4
+msub     x19, x30, xzr, x29
+mneg     x4, x5, x6
+smaddl   x3, w5, w2, x9
+smaddl   xzr, w10, w11, x12
+smaddl   x13, wzr, w14, x15
+smaddl   x16, w17, wzr, x18
+smull    x19, w20, w21
+smsubl   x3, w5, w2, x9
+smsubl   xzr, w10, w11, x12
+smsubl   x13, wzr, w14, x15
+smsubl   x16, w17, wzr, x18
+smnegl   x19, w20, w21
+umaddl   x3, w5, w2, x9
+umaddl   xzr, w10, w11, x12
+umaddl   x13, wzr, w14, x15
+umaddl   x16, w17, wzr, x18
+umull    x19, w20, w21
+umsubl   x3, w5, w2, x9
+umsubl   x16, w17, wzr, x18
+umnegl   x19, w20, w21
+smulh    x30, x29, x28
+smulh    x23, x22, xzr
+umulh    x23, x22, xzr
+mul      x19, x20, xzr
+mneg     w21, w22, w23
+smull    x11, w13, w17
+umull    x11, w13, w17
+smnegl   x11, w13, w17
+umnegl   x11, w13, w17
+
+#------------------------------------------------------------------------------
+# Extract (immediate)
+#------------------------------------------------------------------------------
+
+extr     w3, w5, w7, #0
+extr     w11, w13, w17, #31
+extr     x3, x5, x7, #15
+extr     x11, x13, x17, #63
+ror     x19, x23, #24
+ror     x29, xzr, #63
+ror     w9, w13, #31
+
+#------------------------------------------------------------------------------
+# Floating-point compare
+#------------------------------------------------------------------------------
+
+fcmp    s3, s5
+fcmp    s31, #0.0
+fcmp    s31, #0.0
+fcmpe   s29, s30
+fcmpe   s15, #0.0
+fcmpe   s15, #0.0
+fcmp    d4, d12
+fcmp    d23, #0.0
+fcmp    d23, #0.0
+fcmpe   d26, d22
+fcmpe   d29, #0.0
+fcmpe   d29, #0.0
+
+#------------------------------------------------------------------------------
+# Floating-point conditional compare
+#------------------------------------------------------------------------------
+
+fccmp s1, s31, #0, eq
+fccmp s3, s0, #15, hs
+fccmp s31, s15, #13, hs
+fccmp d9, d31, #0, le
+fccmp d3, d0, #15, gt
+fccmp d31, d5, #7, ne
+fccmpe s1, s31, #0, eq
+fccmpe s3, s0, #15, hs
+fccmpe s31, s15, #13, hs
+fccmpe d9, d31, #0, le
+fccmpe d3, d0, #15, gt
+fccmpe d31, d5, #7, ne
+
+#-------------------------------------------------------------------------------
+# Floating-point conditional compare
+#-------------------------------------------------------------------------------
+
+fcsel s3, s20, s9, pl
+fcsel d9, d10, d11, mi
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmov     s0, s1
+fabs     s2, s3
+fneg     s4, s5
+fsqrt    s6, s7
+fcvt     d8, s9
+fcvt     h10, s11
+frintn   s12, s13
+frintp   s14, s15
+frintm   s16, s17
+frintz   s18, s19
+frinta   s20, s21
+frintx   s22, s23
+frinti   s24, s25
+fmov     d0, d1
+fabs     d2, d3
+fneg     d4, d5
+fsqrt    d6, d7
+fcvt     s8, d9
+fcvt     h10, d11
+frintn   d12, d13
+frintp   d14, d15
+frintm   d16, d17
+frintz   d18, d19
+frinta   d20, d21
+frintx   d22, d23
+frinti   d24, d25
+fcvt     s26, h27
+fcvt     d28, h29
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (2 sources)
+#------------------------------------------------------------------------------
+
+fmul     s20, s19, s17
+fdiv     s1, s2, s3
+fadd     s4, s5, s6
+fsub     s7, s8, s9
+fmax     s10, s11, s12
+fmin     s13, s14, s15
+fmaxnm   s16, s17, s18
+fminnm   s19, s20, s21
+fnmul    s22, s23, s2
+fmul     d20, d19, d17
+fdiv     d1, d2, d3
+fadd     d4, d5, d6
+fsub     d7, d8, d9
+fmax     d10, d11, d12
+fmin     d13, d14, d15
+fmaxnm   d16, d17, d18
+fminnm   d19, d20, d21
+fnmul    d22, d23, d24
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmadd s3, s5, s6, s31
+fmadd d3, d13, d0, d23
+fmsub s3, s5, s6, s31
+fmsub d3, d13, d0, d23
+fnmadd s3, s5, s6, s31
+fnmadd d3, d13, d0, d23
+fnmsub s3, s5, s6, s31
+fnmsub d3, d13, d0, d23
+
+#------------------------------------------------------------------------------
+# Floating-point <-> fixed-point conversion
+#------------------------------------------------------------------------------
+
+fcvtzs  w3, h5, #1
+fcvtzs  wzr, h20, #13
+fcvtzs  w19, h0, #32
+fcvtzs  x3, h5, #1
+fcvtzs  x12, h30, #45
+fcvtzs  x19, h0, #64
+fcvtzs  w3, s5, #1
+fcvtzs  wzr, s20, #13
+fcvtzs  w19, s0, #32
+fcvtzs  x3, s5, #1
+fcvtzs  x12, s30, #45
+fcvtzs  x19, s0, #64
+fcvtzs  w3, d5, #1
+fcvtzs  wzr, d20, #13
+fcvtzs  w19, d0, #32
+fcvtzs  x3, d5, #1
+fcvtzs  x12, d30, #45
+fcvtzs  x19, d0, #64
+fcvtzu  w3, h5, #1
+fcvtzu  wzr, h20, #13
+fcvtzu  w19, h0, #32
+fcvtzu  x3, h5, #1
+fcvtzu  x12, h30, #45
+fcvtzu  x19, h0, #64
+fcvtzu  w3, s5, #1
+fcvtzu  wzr, s20, #13
+fcvtzu  w19, s0, #32
+fcvtzu  x3, s5, #1
+fcvtzu  x12, s30, #45
+fcvtzu  x19, s0, #64
+fcvtzu  w3, d5, #1
+fcvtzu  wzr, d20, #13
+fcvtzu  w19, d0, #32
+fcvtzu  x3, d5, #1
+fcvtzu  x12, d30, #45
+fcvtzu  x19, d0, #64
+scvtf   h23, w19, #1
+scvtf   h31, wzr, #20
+scvtf   h14, w0, #32
+scvtf   h23, x19, #1
+scvtf   h31, xzr, #20
+scvtf   h14, x0, #64
+scvtf   s23, w19, #1
+scvtf   s31, wzr, #20
+scvtf   s14, w0, #32
+scvtf   s23, x19, #1
+scvtf   s31, xzr, #20
+scvtf   s14, x0, #64
+scvtf   d23, w19, #1
+scvtf   d31, wzr, #20
+scvtf   d14, w0, #32
+scvtf   d23, x19, #1
+scvtf   d31, xzr, #20
+scvtf   d14, x0, #64
+ucvtf   h23, w19, #1
+ucvtf   h31, wzr, #20
+ucvtf   h14, w0, #32
+ucvtf   h23, x19, #1
+ucvtf   h31, xzr, #20
+ucvtf   h14, x0, #64
+ucvtf   s23, w19, #1
+ucvtf   s31, wzr, #20
+ucvtf   s14, w0, #32
+ucvtf   s23, x19, #1
+ucvtf   s31, xzr, #20
+ucvtf   s14, x0, #64
+ucvtf   d23, w19, #1
+ucvtf   d31, wzr, #20
+ucvtf   d14, w0, #32
+ucvtf   d23, x19, #1
+ucvtf   d31, xzr, #20
+ucvtf   d14, x0, #64
+
+#------------------------------------------------------------------------------
+# Floating-point <-> integer conversion
+#------------------------------------------------------------------------------
+
+fcvtns   w3, h31
+fcvtns   xzr, h12
+fcvtnu   wzr, h12
+fcvtnu   x0, h0
+fcvtps   wzr, h9
+fcvtps   x12, h20
+fcvtpu   w30, h23
+fcvtpu   x29, h3
+fcvtms   w2, h3
+fcvtms   x4, h5
+fcvtmu   w6, h7
+fcvtmu   x8, h9
+fcvtzs   w10, h11
+fcvtzs   x12, h13
+fcvtzu   w14, h15
+fcvtzu   x15, h16
+scvtf    h17, w18
+scvtf    h19, x20
+ucvtf    h21, w22
+scvtf    h23, x24
+fcvtas   w25, h26
+fcvtas   x27, h28
+fcvtau   w29, h30
+fcvtau   xzr, h0
+fcvtns   w3, s31
+fcvtns   xzr, s12
+fcvtnu   wzr, s12
+fcvtnu   x0, s0
+fcvtps   wzr, s9
+fcvtps   x12, s20
+fcvtpu   w30, s23
+fcvtpu   x29, s3
+fcvtms   w2, s3
+fcvtms   x4, s5
+fcvtmu   w6, s7
+fcvtmu   x8, s9
+fcvtzs   w10, s11
+fcvtzs   x12, s13
+fcvtzu   w14, s15
+fcvtzu   x15, s16
+scvtf    s17, w18
+scvtf    s19, x20
+ucvtf    s21, w22
+scvtf    s23, x24
+fcvtas   w25, s26
+fcvtas   x27, s28
+fcvtau   w29, s30
+fcvtau   xzr, s0
+fcvtns   w3, d31
+fcvtns   xzr, d12
+fcvtnu   wzr, d12
+fcvtnu   x0, d0
+fcvtps   wzr, d9
+fcvtps   x12, d20
+fcvtpu   w30, d23
+fcvtpu   x29, d3
+fcvtms   w2, d3
+fcvtms   x4, d5
+fcvtmu   w6, d7
+fcvtmu   x8, d9
+fcvtzs   w10, d11
+fcvtzs   x12, d13
+fcvtzu   w14, d15
+fcvtzu   x15, d16
+scvtf    d17, w18
+scvtf    d19, x20
+ucvtf    d21, w22
+ucvtf    d23, x24
+fcvtas   w25, d26
+fcvtas   x27, d28
+fcvtau   w29, d30
+fcvtau   xzr, d0
+fmov     w3, s9
+fmov     s9, w3
+fmov     x20, d31
+fmov     d1, x15
+fmov     x3, v12.d[1]
+fmov     v1.d[1], x19
+
+#------------------------------------------------------------------------------
+# Floating-point immediate
+#------------------------------------------------------------------------------
+
+fmov     s2, #0.12500000
+fmov     s3, #1.00000000
+fmov     d30, #16.00000000
+fmov     s4, #1.06250000
+fmov     d10, #1.93750000
+fmov     s12, #-1.00000000
+fmov     d16, #8.50000000
+
+#------------------------------------------------------------------------------
+# Load-register (literal)
+#------------------------------------------------------------------------------
+
+ldr       w3, #0
+ldr       x29, #4
+ldrsw     xzr, #-4
+ldr       s0, #8
+ldr       d0, #1048572
+ldr       q0, #-1048576
+prfm      pldl1strm, #0
+prfm      #22, #0
+
+#------------------------------------------------------------------------------
+# Load/store exclusive
+#------------------------------------------------------------------------------
+
+stxrb      w18, w8, [sp]
+stxrh      w24, w15, [x16]
+stxr       w5, w6, [x17]
+stxr       w1, x10, [x21]
+ldxrb      w30, [x0]
+ldxrh      w17, [x4]
+ldxr       w22, [sp]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+stxp       w12, w11, w10, [sp]
+stxp       wzr, x27, x9, [x12]
+ldxp       w0, wzr, [sp]
+ldxp       x17, x0, [x18]
+ldxp       x17, x0, [x18]
+stlxrb     w12, w22, [x0]
+stlxrh     w10, w1, [x1]
+stlxr      w9, w2, [x2]
+stlxr      w9, x3, [sp]
+ldaxrb     w8, [x4]
+ldaxrh     w7, [x5]
+ldaxr      w6, [sp]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+stlxp      w4, w5, w6, [sp]
+stlxp      wzr, x6, x7, [x1]
+ldaxp      w5, w18, [sp]
+ldaxp      x6, x19, [x22]
+ldaxp      x6, x19, [x22]
+stlrb      w24, [sp]
+stlrh      w25, [x30]
+stlr       w26, [x29]
+stlr       x27, [x28]
+stlr       x27, [x28]
+stlr       x27, [x28]
+ldarb      w23, [sp]
+ldarh      w22, [x30]
+ldar       wzr, [x29]
+ldar       x21, [x28]
+ldar       x21, [x28]
+ldar       x21, [x28]
+
+#------------------------------------------------------------------------------
+# Load/store (unscaled  immediate)
+#------------------------------------------------------------------------------
+
+sturb    w9, [sp]
+sturh    wzr, [x12, #255]
+stur     w16, [x0, #-256]
+stur     x28, [x14, #1]
+ldurb    w1, [x20, #255]
+ldurh    w20, [x1, #255]
+ldur     w12, [sp, #255]
+ldur     xzr, [x12, #255]
+ldursb   x9, [x7, #-256]
+ldursh   x17, [x19, #-256]
+ldursw   x20, [x15, #-256]
+prfum    pldl2keep, [sp, #-256]
+ldursb   w19, [x1, #-256]
+ldursh   w15, [x21, #-256]
+stur     b0, [sp, #1]
+stur     h12, [x12, #-1]
+stur     s15, [x0, #255]
+stur     d31, [x5, #25]
+stur     q9, [x5]
+ldur     b3, [sp]
+ldur     h5, [x4, #-256]
+ldur     s7, [x12, #-1]
+ldur     d11, [x19, #4]
+ldur     q13, [x1, #2]
+
+#------------------------------------------------------------------------------
+# Load/store (immediate post-indexed)
+#------------------------------------------------------------------------------
+
+strb     w9, [x2], #255
+strb     w10, [x3], #1
+strb     w10, [x3], #-256
+strh     w9, [x2], #255
+strh     w9, [x2], #1
+strh     w10, [x3], #-256
+str      w19, [sp], #255
+str      w20, [x30], #1
+str      w21, [x12], #-256
+str      xzr, [x9], #255
+str      x2, [x3], #1
+str      x19, [x12], #-256
+ldrb     w9, [x2], #255
+ldrb     w10, [x3], #1
+ldrb     w10, [x3], #-256
+ldrh     w9, [x2], #255
+ldrh     w9, [x2], #1
+ldrh     w10, [x3], #-256
+ldr      w19, [sp], #255
+ldr      w20, [x30], #1
+ldr      w21, [x12], #-256
+ldr      xzr, [x9], #255
+ldr      x2, [x3], #1
+ldr      x19, [x12], #-256
+ldrsb    xzr, [x9], #255
+ldrsb    x2, [x3], #1
+ldrsb    x19, [x12], #-256
+ldrsh    xzr, [x9], #255
+ldrsh    x2, [x3], #1
+ldrsh    x19, [x12], #-256
+ldrsw    xzr, [x9], #255
+ldrsw    x2, [x3], #1
+ldrsw    x19, [x12], #-256
+ldrsb    wzr, [x9], #255
+ldrsb    w2, [x3], #1
+ldrsb    w19, [x12], #-256
+ldrsh    wzr, [x9], #255
+ldrsh    w2, [x3], #1
+ldrsh    w19, [x12], #-256
+str      b0, [x0], #255
+str      b3, [x3], #1
+str      b5, [sp], #-256
+str      h10, [x10], #255
+str      h13, [x23], #1
+str      h15, [sp], #-256
+str      s20, [x20], #255
+str      s23, [x23], #1
+str      s25, [x0], #-256
+str      d20, [x20], #255
+str      d23, [x23], #1
+str      d25, [x0], #-256
+ldr      b0, [x0], #255
+ldr      b3, [x3], #1
+ldr      b5, [sp], #-256
+ldr      h10, [x10], #255
+ldr      h13, [x23], #1
+ldr      h15, [sp], #-256
+ldr      s20, [x20], #255
+ldr      s23, [x23], #1
+ldr      s25, [x0], #-256
+ldr      d20, [x20], #255
+ldr      d23, [x23], #1
+ldr      d25, [x0], #-256
+ldr      q20, [x1], #255
+ldr      q23, [x9], #1
+ldr      q25, [x20], #-256
+str      q10, [x1], #255
+str      q22, [sp], #1
+str      q21, [x20], #-256
+
+#-------------------------------------------------------------------------------
+# Load-store register (immediate pre-indexed)
+#-------------------------------------------------------------------------------
+
+ldr      x3, [x4, #0]!
+strb     w9, [x2, #255]!
+strb     w10, [x3, #1]!
+strb     w10, [x3, #-256]!
+strh     w9, [x2, #255]!
+strh     w9, [x2, #1]!
+strh     w10, [x3, #-256]!
+str      w19, [sp, #255]!
+str      w20, [x30, #1]!
+str      w21, [x12, #-256]!
+str      xzr, [x9, #255]!
+str      x2, [x3, #1]!
+str      x19, [x12, #-256]!
+ldrb     w9, [x2, #255]!
+ldrb     w10, [x3, #1]!
+ldrb     w10, [x3, #-256]!
+ldrh     w9, [x2, #255]!
+ldrh     w9, [x2, #1]!
+ldrh     w10, [x3, #-256]!
+ldr      w19, [sp, #255]!
+ldr      w20, [x30, #1]!
+ldr      w21, [x12, #-256]!
+ldr      xzr, [x9, #255]!
+ldr      x2, [x3, #1]!
+ldr      x19, [x12, #-256]!
+ldrsb    xzr, [x9, #255]!
+ldrsb    x2, [x3, #1]!
+ldrsb    x19, [x12, #-256]!
+ldrsh    xzr, [x9, #255]!
+ldrsh    x2, [x3, #1]!
+ldrsh    x19, [x12, #-256]!
+ldrsw    xzr, [x9, #255]!
+ldrsw    x2, [x3, #1]!
+ldrsw    x19, [x12, #-256]!
+ldrsb    wzr, [x9, #255]!
+ldrsb    w2, [x3, #1]!
+ldrsb    w19, [x12, #-256]!
+ldrsh    wzr, [x9, #255]!
+ldrsh    w2, [x3, #1]!
+ldrsh    w19, [x12, #-256]!
+str      b0, [x0, #255]!
+str      b3, [x3, #1]!
+str      b5, [sp, #-256]!
+str      h10, [x10, #255]!
+str      h13, [x23, #1]!
+str      h15, [sp, #-256]!
+str      s20, [x20, #255]!
+str      s23, [x23, #1]!
+str      s25, [x0, #-256]!
+str      d20, [x20, #255]!
+str      d23, [x23, #1]!
+str      d25, [x0, #-256]!
+ldr      b0, [x0, #255]!
+ldr      b3, [x3, #1]!
+ldr      b5, [sp, #-256]!
+ldr      h10, [x10, #255]!
+ldr      h13, [x23, #1]!
+ldr      h15, [sp, #-256]!
+ldr      s20, [x20, #255]!
+ldr      s23, [x23, #1]!
+ldr      s25, [x0, #-256]!
+ldr      d20, [x20, #255]!
+ldr      d23, [x23, #1]!
+ldr      d25, [x0, #-256]!
+ldr      q20, [x1, #255]!
+ldr      q23, [x9, #1]!
+ldr      q25, [x20, #-256]!
+str      q10, [x1, #255]!
+str      q22, [sp, #1]!
+str      q21, [x20, #-256]!
+
+#------------------------------------------------------------------------------
+# Load/store (unprivileged)
+#------------------------------------------------------------------------------
+
+sttrb    w9, [sp]
+sttrh    wzr, [x12, #255]
+sttr     w16, [x0, #-256]
+sttr     x28, [x14, #1]
+ldtrb    w1, [x20, #255]
+ldtrh    w20, [x1, #255]
+ldtr     w12, [sp, #255]
+ldtr     xzr, [x12, #255]
+ldtrsb   x9, [x7, #-256]
+ldtrsh   x17, [x19, #-256]
+ldtrsw   x20, [x15, #-256]
+ldtrsb   w19, [x1, #-256]
+ldtrsh   w15, [x21, #-256]
+
+#------------------------------------------------------------------------------
+# Load/store (unsigned  immediate)
+#------------------------------------------------------------------------------
+
+ldr      x4, [x29]
+ldr      x30, [x12, #32760]
+ldr      x20, [sp, #8]
+ldr      xzr, [sp]
+ldr      w2, [sp]
+ldr      w17, [sp, #16380]
+ldr      w13, [x2, #4]
+ldrsw    x2, [x5, #4]
+ldrsw    x23, [sp, #16380]
+ldrh     w2, [x4]
+ldrsh    w23, [x6, #8190]
+ldrsh    wzr, [sp, #2]
+ldrsh    x29, [x2, #2]
+ldrb     w26, [x3, #121]
+ldrb     w12, [x2]
+ldrsb    w27, [sp, #4095]
+ldrsb    xzr, [x15]
+str      x30, [sp]
+str      w20, [x4, #16380]
+strh     w17, [sp, #8190]
+strb     w23, [x3, #4095]
+strb     wzr, [x2]
+ldr      b31, [sp, #4095]
+ldr      h20, [x2, #8190]
+ldr      s10, [x19, #16380]
+ldr      d3, [x10, #32760]
+str      q12, [sp, #65520]
+
+#------------------------------------------------------------------------------
+# Load/store (register offset)
+#------------------------------------------------------------------------------
+
+ldrb     w3, [sp, x5]
+ldrb     w9, [x27, x6]
+ldrsb    w10, [x30, x7]
+ldrb     w11, [x29, x3, sxtx]
+strb     w12, [x28, xzr, sxtx]
+ldrb     w14, [x26, w6, uxtw]
+ldrsb    w15, [x25, w7, uxtw]
+ldrb     w17, [x23, w9, sxtw]
+ldrsb    x18, [x22, w10, sxtw]
+ldrsh    w3, [sp, x5]
+ldrsh    w9, [x27, x6]
+ldrh     w10, [x30, x7, lsl #1]
+strh     w11, [x29, x3, sxtx]
+ldrh     w12, [x28, xzr, sxtx]
+ldrsh    x13, [x27, x5, sxtx #1]
+ldrh     w14, [x26, w6, uxtw]
+ldrh     w15, [x25, w7, uxtw]
+ldrsh    w16, [x24, w8, uxtw #1]
+ldrh     w17, [x23, w9, sxtw]
+ldrh     w18, [x22, w10, sxtw]
+strh     w19, [x21, wzr, sxtw #1]
+ldr      w3, [sp, x5]
+ldr      s9, [x27, x6]
+ldr      w10, [x30, x7, lsl #2]
+ldr      w11, [x29, x3, sxtx]
+str      s12, [x28, xzr, sxtx]
+str      w13, [x27, x5, sxtx #2]
+str      w14, [x26, w6, uxtw]
+ldr      w15, [x25, w7, uxtw]
+ldr      w16, [x24, w8, uxtw #2]
+ldrsw    x17, [x23, w9, sxtw]
+ldr      w18, [x22, w10, sxtw]
+ldrsw    x19, [x21, wzr, sxtw #2]
+ldr      x3, [sp, x5]
+str      x9, [x27, x6]
+ldr      d10, [x30, x7, lsl #3]
+str      x11, [x29, x3, sxtx]
+ldr      x12, [x28, xzr, sxtx]
+ldr      x13, [x27, x5, sxtx #3]
+prfm     pldl1keep, [x26, w6, uxtw]
+ldr      x15, [x25, w7, uxtw]
+ldr      x16, [x24, w8, uxtw #3]
+ldr      x17, [x23, w9, sxtw]
+ldr      x18, [x22, w10, sxtw]
+str      d19, [x21, wzr, sxtw #3]
+ldr      q3, [sp, x5]
+ldr      q9, [x27, x6]
+ldr      q10, [x30, x7, lsl #4]
+str      q11, [x29, x3, sxtx]
+str      q12, [x28, xzr, sxtx]
+str      q13, [x27, x5, sxtx #4]
+ldr      q14, [x26, w6, uxtw]
+ldr      q15, [x25, w7, uxtw]
+ldr      q16, [x24, w8, uxtw #4]
+ldr      q17, [x23, w9, sxtw]
+str      q18, [x22, w10, sxtw]
+ldr      q19, [x21, wzr, sxtw #4]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp]
+stp      wzr, w9, [sp, #252]
+ldp      w2, wzr, [sp, #-256]
+ldp      w9, w10, [sp, #4]
+ldpsw    x9, x10, [sp, #4]
+ldpsw    x9, x10, [x2, #-256]
+ldpsw    x20, x30, [sp, #252]
+ldp      x21, x29, [x2, #504]
+ldp      x22, x23, [x3, #-512]
+ldp      x24, x25, [x4, #8]
+ldp      s29, s28, [sp, #252]
+stp      s27, s26, [sp, #-256]
+ldp      s1, s2, [x3, #44]
+stp      d3, d5, [x9, #504]
+stp      d7, d11, [x10, #-512]
+ldp      d2, d3, [x30, #-8]
+stp      q3, q5, [sp]
+stp      q17, q19, [sp, #1008]
+ldp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (post-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp], #0
+stp      wzr, w9, [sp], #252
+ldp      w2, wzr, [sp], #-256
+ldp      w9, w10, [sp], #4
+ldpsw    x9, x10, [sp], #4
+ldpsw    x9, x10, [x2], #-256
+ldpsw    x20, x30, [sp], #252
+ldp      x21, x29, [x2], #504
+ldp      x22, x23, [x3], #-512
+ldp      x24, x25, [x4], #8
+ldp      s29, s28, [sp], #252
+stp      s27, s26, [sp], #-256
+ldp      s1, s2, [x3], #44
+stp      d3, d5, [x9], #504
+stp      d7, d11, [x10], #-512
+ldp      d2, d3, [x30], #-8
+stp      q3, q5, [sp], #0
+stp      q17, q19, [sp], #1008
+ldp      q23, q29, [x1], #-1024
+
+#------------------------------------------------------------------------------
+# Load/store register pair (pre-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp, #0]!
+stp      wzr, w9, [sp, #252]!
+ldp      w2, wzr, [sp, #-256]!
+ldp      w9, w10, [sp, #4]!
+ldpsw    x9, x10, [sp, #4]!
+ldpsw    x9, x10, [x2, #-256]!
+ldpsw    x20, x30, [sp, #252]!
+ldp      x21, x29, [x2, #504]!
+ldp      x22, x23, [x3, #-512]!
+ldp      x24, x25, [x4, #8]!
+ldp      s29, s28, [sp, #252]!
+stp      s27, s26, [sp, #-256]!
+ldp      s1, s2, [x3, #44]!
+stp      d3, d5, [x9, #504]!
+stp      d7, d11, [x10, #-512]!
+ldp      d2, d3, [x30, #-8]!
+stp      q3, q5, [sp, #0]!
+stp      q17, q19, [sp, #1008]!
+ldp      q23, q29, [x1, #-1024]!
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldnp      w3, w5, [sp]
+stnp      wzr, w9, [sp, #252]
+ldnp      w2, wzr, [sp, #-256]
+ldnp      w9, w10, [sp, #4]
+ldnp      x21, x29, [x2, #504]
+ldnp      x22, x23, [x3, #-512]
+ldnp      x24, x25, [x4, #8]
+ldnp      s29, s28, [sp, #252]
+stnp      s27, s26, [sp, #-256]
+ldnp      s1, s2, [x3, #44]
+stnp      d3, d5, [x9, #504]
+stnp      d7, d11, [x10, #-512]
+ldnp      d2, d3, [x30, #-8]
+stnp      q3, q5, [sp]
+stnp      q17, q19, [sp, #1008]
+ldnp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Logical (immediate)
+#------------------------------------------------------------------------------
+
+mov      w3, #983055
+mov      x10, #-6148914691236517206
+
+#------------------------------------------------------------------------------
+# Logical (shifted register)
+#------------------------------------------------------------------------------
+
+and      w12, w23, w21
+and      w16, w15, w1, lsl #1
+and      w9, w4, w10, lsl #31
+and      w3, w30, w11
+and      x3, x5, x7, lsl #63
+and      x5, x14, x19, asr #4
+and      w3, w17, w19, ror #31
+and      w0, w2, wzr, lsr #17
+and      w3, w30, w11, asr #2
+and      xzr, x4, x26
+and      w3, wzr, w20, ror #2
+and      x7, x20, xzr, asr #63
+bic      x13, x20, x14, lsl #47
+bic      w2, w7, w9
+orr      w2, w7, w0, asr #31
+orr      x8, x9, x10, lsl #12
+orn      x3, x5, x7, asr #2
+orn      w2, w5, w29
+ands     w7, wzr, w9, lsl #1
+ands     x3, x5, x20, ror #63
+bics     w3, w5, w7
+bics     x3, xzr, x3, lsl #1
+tst      w3, w7, lsl #31
+tst      x2, x20, asr #2
+mov      x3, x6
+mov      x3, xzr
+mov      wzr, w2
+mov      w3, w5
+
+#------------------------------------------------------------------------------
+# Move wide (immediate)
+#------------------------------------------------------------------------------
+
+movz     w2, #0, lsl #16
+mov     w2, #-1235
+mov     x2, #5299989643264
+mov      x2, #0
+movk     w3, #0
+movz     x4, #0, lsl #16
+movk     w5, #0, lsl #16
+movz     x6, #0, lsl #32
+movk     x7, #0, lsl #32
+movz     x8, #0, lsl #48
+movk     x9, #0, lsl #48
+
+#------------------------------------------------------------------------------
+# PC-relative addressing
+#------------------------------------------------------------------------------
+
+adr      x2, #1600
+adrp     x21, #6553600
+adr      x0, #262144
+
+#------------------------------------------------------------------------------
+# Test and branch (immediate)
+#------------------------------------------------------------------------------
+
+tbz     x12, #62, #0
+tbz     x12, #62, #4
+tbz     x12, #62, #-32768
+tbnz    x12, #60, #32764
+
+#------------------------------------------------------------------------------
+# Unconditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b        #4
+b        #-4
+b        #134217724
+
+#------------------------------------------------------------------------------
+# Unconditional branch (register)
+#------------------------------------------------------------------------------
+
+br       x20
+blr      xzr
+ret      x10
+ret
+eret
+drps
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     1.00                        add	w2, w3, #4095
+# CHECK-NEXT:  1      1     1.00                        add	w30, w29, #1, lsl #12
+# CHECK-NEXT:  1      1     1.00                        add	w13, w5, #4095, lsl #12
+# CHECK-NEXT:  1      1     1.00                        add	x5, x7, #1638
+# CHECK-NEXT:  1      1     1.00                        add	w20, wsp, #801
+# CHECK-NEXT:  1      1     1.00                        add	wsp, wsp, #1104
+# CHECK-NEXT:  1      1     1.00                        add	wsp, w30, #4084
+# CHECK-NEXT:  1      1     1.00                        add	x0, x24, #291
+# CHECK-NEXT:  1      1     1.00                        add	x3, x24, #4095, lsl #12
+# CHECK-NEXT:  1      1     1.00                        add	x8, sp, #1074
+# CHECK-NEXT:  1      1     1.00                        add	sp, x29, #3816
+# CHECK-NEXT:  1      1     1.00                        sub	w0, wsp, #4077
+# CHECK-NEXT:  1      1     1.00                        sub	w4, w20, #546, lsl #12
+# CHECK-NEXT:  1      1     1.00                        sub	sp, sp, #288
+# CHECK-NEXT:  1      1     1.00                        sub	wsp, w19, #16
+# CHECK-NEXT:  1      1     1.00                        adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  1      1     1.00                        cmn	w2, #4095
+# CHECK-NEXT:  1      1     1.00                        adds	w20, wsp, #0
+# CHECK-NEXT:  1      1     1.00                        cmn	x3, #1, lsl #12
+# CHECK-NEXT:  1      1     1.00                        cmp	sp, #20, lsl #12
+# CHECK-NEXT:  1      1     1.00                        cmp	x30, #4095
+# CHECK-NEXT:  1      1     1.00                        subs	x4, sp, #3822
+# CHECK-NEXT:  1      1     1.00                        cmn	w3, #291, lsl #12
+# CHECK-NEXT:  1      1     1.00                        cmn	wsp, #1365
+# CHECK-NEXT:  1      1     1.00                        cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  1      1     1.00                        mov	sp, x30
+# CHECK-NEXT:  1      1     1.00                        mov	wsp, w20
+# CHECK-NEXT:  1      1     1.00                        mov	x11, sp
+# CHECK-NEXT:  1      1     1.00                        mov	w24, wsp
+# CHECK-NEXT:  1      1     1.00                        add	w3, w5, w7
+# CHECK-NEXT:  1      1     1.00                        add	wzr, w3, w5
+# CHECK-NEXT:  1      1     1.00                        add	w20, wzr, w4
+# CHECK-NEXT:  1      1     1.00                        add	w4, w6, wzr
+# CHECK-NEXT:  1      1     1.00                        add	w11, w13, w15
+# CHECK-NEXT:  1      2     1.00                        add	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        add	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     1.00                        add	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        add	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        add	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     1.00                        add	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     1.00                        add	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     1.00                        add	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     1.00                        add	x3, x5, x7
+# CHECK-NEXT:  1      1     1.00                        add	xzr, x3, x5
+# CHECK-NEXT:  1      1     1.00                        add	x20, xzr, x4
+# CHECK-NEXT:  1      1     1.00                        add	x4, x6, xzr
+# CHECK-NEXT:  1      1     1.00                        add	x11, x13, x15
+# CHECK-NEXT:  1      2     1.00                        add	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        add	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     1.00                        add	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        add	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        add	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     1.00                        add	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     1.00                        add	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     1.00                        add	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     1.00                        adds	w3, w5, w7
+# CHECK-NEXT:  1      1     1.00                        cmn	w3, w5
+# CHECK-NEXT:  1      1     1.00                        adds	w20, wzr, w4
+# CHECK-NEXT:  1      1     1.00                        adds	w4, w6, wzr
+# CHECK-NEXT:  1      1     1.00                        adds	w11, w13, w15
+# CHECK-NEXT:  1      2     1.00                        adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        adds	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     1.00                        adds	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        adds	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        adds	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     1.00                        adds	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     1.00                        adds	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     1.00                        adds	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     1.00                        adds	x3, x5, x7
+# CHECK-NEXT:  1      1     1.00                        cmn	x3, x5
+# CHECK-NEXT:  1      1     1.00                        adds	x20, xzr, x4
+# CHECK-NEXT:  1      1     1.00                        adds	x4, x6, xzr
+# CHECK-NEXT:  1      1     1.00                        adds	x11, x13, x15
+# CHECK-NEXT:  1      2     1.00                        adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        adds	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     1.00                        adds	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        adds	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        adds	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     1.00                        adds	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     1.00                        adds	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     1.00                        adds	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     1.00                        sub	w3, w5, w7
+# CHECK-NEXT:  1      1     1.00                        sub	wzr, w3, w5
+# CHECK-NEXT:  1      1     1.00                        sub	w4, w6, wzr
+# CHECK-NEXT:  1      1     1.00                        sub	w11, w13, w15
+# CHECK-NEXT:  1      2     1.00                        sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        sub	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     1.00                        sub	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        sub	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        sub	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     1.00                        sub	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     1.00                        sub	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     1.00                        sub	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     1.00                        sub	x3, x5, x7
+# CHECK-NEXT:  1      1     1.00                        sub	xzr, x3, x5
+# CHECK-NEXT:  1      1     1.00                        sub	x4, x6, xzr
+# CHECK-NEXT:  1      1     1.00                        sub	x11, x13, x15
+# CHECK-NEXT:  1      2     1.00                        sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        sub	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     1.00                        sub	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        sub	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        sub	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     1.00                        sub	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     1.00                        sub	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     1.00                        sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     1.00                        subs	w3, w5, w7
+# CHECK-NEXT:  1      1     1.00                        cmp	w3, w5
+# CHECK-NEXT:  1      1     1.00                        subs	w4, w6, wzr
+# CHECK-NEXT:  1      1     1.00                        subs	w11, w13, w15
+# CHECK-NEXT:  1      2     1.00                        subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        subs	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     1.00                        subs	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        subs	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        subs	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     1.00                        subs	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     1.00                        subs	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     1.00                        subs	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     1.00                        subs	x3, x5, x7
+# CHECK-NEXT:  1      1     1.00                        cmp	x3, x5
+# CHECK-NEXT:  1      1     1.00                        subs	x4, x6, xzr
+# CHECK-NEXT:  1      1     1.00                        subs	x11, x13, x15
+# CHECK-NEXT:  1      2     1.00                        subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     1.00                        subs	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     1.00                        subs	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     1.00                        subs	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     1.00                        subs	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     1.00                        subs	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     1.00                        subs	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     1.00                        subs	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     1.00                        cmn	wzr, w4
+# CHECK-NEXT:  1      1     1.00                        cmn	w5, wzr
+# CHECK-NEXT:  1      1     1.00                        cmn	w6, w7
+# CHECK-NEXT:  1      2     1.00                        cmn	w8, w9, lsl #15
+# CHECK-NEXT:  1      2     1.00                        cmn	w10, w11, lsl #31
+# CHECK-NEXT:  1      2     1.00                        cmn	w12, w13, lsr #0
+# CHECK-NEXT:  1      2     1.00                        cmn	w14, w15, lsr #21
+# CHECK-NEXT:  1      2     1.00                        cmn	w16, w17, lsr #31
+# CHECK-NEXT:  1      2     1.00                        cmn	w18, w19, asr #0
+# CHECK-NEXT:  1      2     1.00                        cmn	w20, w21, asr #22
+# CHECK-NEXT:  1      2     1.00                        cmn	w22, w23, asr #31
+# CHECK-NEXT:  1      1     1.00                        cmn	x0, x3
+# CHECK-NEXT:  1      1     1.00                        cmn	xzr, x4
+# CHECK-NEXT:  1      1     1.00                        cmn	x5, xzr
+# CHECK-NEXT:  1      1     1.00                        cmn	x6, x7
+# CHECK-NEXT:  1      2     1.00                        cmn	x8, x9, lsl #15
+# CHECK-NEXT:  1      2     1.00                        cmn	x10, x11, lsl #63
+# CHECK-NEXT:  1      2     1.00                        cmn	x12, x13, lsr #0
+# CHECK-NEXT:  1      2     1.00                        cmn	x14, x15, lsr #41
+# CHECK-NEXT:  1      2     1.00                        cmn	x16, x17, lsr #63
+# CHECK-NEXT:  1      2     1.00                        cmn	x18, x19, asr #0
+# CHECK-NEXT:  1      2     1.00                        cmn	x20, x21, asr #55
+# CHECK-NEXT:  1      2     1.00                        cmn	x22, x23, asr #63
+# CHECK-NEXT:  1      1     1.00                        cmp	w0, w3
+# CHECK-NEXT:  1      1     1.00                        cmp	wzr, w4
+# CHECK-NEXT:  1      1     1.00                        cmp	w5, wzr
+# CHECK-NEXT:  1      1     1.00                        cmp	w6, w7
+# CHECK-NEXT:  1      2     1.00                        cmp	w8, w9, lsl #15
+# CHECK-NEXT:  1      2     1.00                        cmp	w10, w11, lsl #31
+# CHECK-NEXT:  1      2     1.00                        cmp	w12, w13, lsr #0
+# CHECK-NEXT:  1      2     1.00                        cmp	w14, w15, lsr #21
+# CHECK-NEXT:  1      2     1.00                        cmp	w18, w19, asr #0
+# CHECK-NEXT:  1      2     1.00                        cmp	w20, w21, asr #22
+# CHECK-NEXT:  1      2     1.00                        cmp	w22, w23, asr #31
+# CHECK-NEXT:  1      1     1.00                        cmp	x0, x3
+# CHECK-NEXT:  1      1     1.00                        cmp	xzr, x4
+# CHECK-NEXT:  1      1     1.00                        cmp	x5, xzr
+# CHECK-NEXT:  1      1     1.00                        cmp	x6, x7
+# CHECK-NEXT:  1      2     1.00                        cmp	x8, x9, lsl #15
+# CHECK-NEXT:  1      2     1.00                        cmp	x10, x11, lsl #63
+# CHECK-NEXT:  1      2     1.00                        cmp	x12, x13, lsr #0
+# CHECK-NEXT:  1      2     1.00                        cmp	x14, x15, lsr #41
+# CHECK-NEXT:  1      2     1.00                        cmp	x16, x17, lsr #63
+# CHECK-NEXT:  1      2     1.00                        cmp	x18, x19, asr #0
+# CHECK-NEXT:  1      2     1.00                        cmp	x20, x21, asr #55
+# CHECK-NEXT:  1      2     1.00                        cmp	x22, x23, asr #63
+# CHECK-NEXT:  1      1     1.00                        cmp	wzr, w0
+# CHECK-NEXT:  1      1     1.00                        cmp	xzr, x0
+# CHECK-NEXT:  1      1     1.00                        adc	w29, w27, w25
+# CHECK-NEXT:  1      1     1.00                        adc	wzr, w3, w4
+# CHECK-NEXT:  1      1     1.00                        adc	w9, wzr, w10
+# CHECK-NEXT:  1      1     1.00                        adc	w20, w0, wzr
+# CHECK-NEXT:  1      1     1.00                        adc	x29, x27, x25
+# CHECK-NEXT:  1      1     1.00                        adc	xzr, x3, x4
+# CHECK-NEXT:  1      1     1.00                        adc	x9, xzr, x10
+# CHECK-NEXT:  1      1     1.00                        adc	x20, x0, xzr
+# CHECK-NEXT:  1      1     1.00                        adcs	w29, w27, w25
+# CHECK-NEXT:  1      1     1.00                        adcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     1.00                        adcs	w9, wzr, w10
+# CHECK-NEXT:  1      1     1.00                        adcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     1.00                        adcs	x29, x27, x25
+# CHECK-NEXT:  1      1     1.00                        adcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     1.00                        adcs	x9, xzr, x10
+# CHECK-NEXT:  1      1     1.00                        adcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     1.00                        sbc	w29, w27, w25
+# CHECK-NEXT:  1      1     1.00                        sbc	wzr, w3, w4
+# CHECK-NEXT:  1      1     1.00                        ngc	w9, w10
+# CHECK-NEXT:  1      1     1.00                        sbc	w20, w0, wzr
+# CHECK-NEXT:  1      1     1.00                        sbc	x29, x27, x25
+# CHECK-NEXT:  1      1     1.00                        sbc	xzr, x3, x4
+# CHECK-NEXT:  1      1     1.00                        ngc	x9, x10
+# CHECK-NEXT:  1      1     1.00                        sbc	x20, x0, xzr
+# CHECK-NEXT:  1      1     1.00                        sbcs	w29, w27, w25
+# CHECK-NEXT:  1      1     1.00                        sbcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     1.00                        ngcs	w9, w10
+# CHECK-NEXT:  1      1     1.00                        sbcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     1.00                        sbcs	x29, x27, x25
+# CHECK-NEXT:  1      1     1.00                        sbcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     1.00                        ngcs	x9, x10
+# CHECK-NEXT:  1      1     1.00                        sbcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     1.00                        ngc	w3, w12
+# CHECK-NEXT:  1      1     1.00                        ngc	wzr, w9
+# CHECK-NEXT:  1      1     1.00                        ngc	w23, wzr
+# CHECK-NEXT:  1      1     1.00                        ngc	x29, x30
+# CHECK-NEXT:  1      1     1.00                        ngc	xzr, x0
+# CHECK-NEXT:  1      1     1.00                        ngc	x0, xzr
+# CHECK-NEXT:  1      1     1.00                        ngcs	w3, w12
+# CHECK-NEXT:  1      1     1.00                        ngcs	wzr, w9
+# CHECK-NEXT:  1      1     1.00                        ngcs	w23, wzr
+# CHECK-NEXT:  1      1     1.00                        ngcs	x29, x30
+# CHECK-NEXT:  1      1     1.00                        ngcs	xzr, x0
+# CHECK-NEXT:  1      1     1.00                        ngcs	x0, xzr
+# CHECK-NEXT:  1      2     1.00                        sbfx	x1, x2, #3, #2
+# CHECK-NEXT:  1      2     1.00                        asr	x3, x4, #63
+# CHECK-NEXT:  1      2     1.00                        asr	wzr, wzr, #31
+# CHECK-NEXT:  1      2     1.00                        sbfx	w12, w9, #0, #1
+# CHECK-NEXT:  1      2     1.00                        ubfiz	x4, x5, #52, #11
+# CHECK-NEXT:  1      2     1.00                        ubfx	xzr, x4, #0, #1
+# CHECK-NEXT:  1      2     1.00                        ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT:  1      2     1.00                        lsr	x5, x6, #12
+# CHECK-NEXT:  1      2     1.00                        bfi	x4, x5, #52, #11
+# CHECK-NEXT:  1      2     1.00                        bfxil	xzr, x4, #0, #1
+# CHECK-NEXT:  1      2     1.00                        bfc	x4, #1, #6
+# CHECK-NEXT:  1      2     1.00                        bfxil	x5, x6, #12, #52
+# CHECK-NEXT:  1      2     1.00                        sxtb	w1, w2
+# CHECK-NEXT:  1      2     1.00                        sxtb	xzr, w3
+# CHECK-NEXT:  1      2     1.00                        sxth	w9, w10
+# CHECK-NEXT:  1      2     1.00                        sxth	x0, w1
+# CHECK-NEXT:  1      2     1.00                        sxtw	x3, w30
+# CHECK-NEXT:  1      2     1.00                        uxtb	w1, w2
+# CHECK-NEXT:  1      2     1.00                        uxth	w9, w10
+# CHECK-NEXT:  1      2     1.00                        ubfx	x3, x30, #0, #32
+# CHECK-NEXT:  1      2     1.00                        asr	w3, w2, #0
+# CHECK-NEXT:  1      2     1.00                        asr	w9, w10, #31
+# CHECK-NEXT:  1      2     1.00                        asr	x20, x21, #63
+# CHECK-NEXT:  1      2     1.00                        asr	w1, wzr, #3
+# CHECK-NEXT:  1      2     1.00                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      2     1.00                        lsr	w9, w10, #31
+# CHECK-NEXT:  1      2     1.00                        lsr	x20, x21, #63
+# CHECK-NEXT:  1      2     1.00                        lsr	wzr, wzr, #3
+# CHECK-NEXT:  1      2     1.00                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      2     1.00                        lsl	w9, w10, #31
+# CHECK-NEXT:  1      2     1.00                        lsl	x20, x21, #63
+# CHECK-NEXT:  1      2     1.00                        lsl	w1, wzr, #3
+# CHECK-NEXT:  1      2     1.00                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        sbfiz	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     1.00                        asr	x19, x20, #0
+# CHECK-NEXT:  1      2     1.00                        sbfiz	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     1.00                        asr	w9, w10, #0
+# CHECK-NEXT:  1      2     1.00                        sbfiz	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     1.00                        sbfiz	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     1.00                        sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     1.00                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        asr	x2, x3, #63
+# CHECK-NEXT:  1      2     1.00                        asr	x19, x20, #0
+# CHECK-NEXT:  1      2     1.00                        asr	x9, x10, #5
+# CHECK-NEXT:  1      2     1.00                        asr	w9, w10, #0
+# CHECK-NEXT:  1      2     1.00                        asr	w11, w12, #31
+# CHECK-NEXT:  1      2     1.00                        asr	w13, w14, #29
+# CHECK-NEXT:  1      2     1.00                        sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     1.00                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        bfi	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     1.00                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      2     1.00                        bfi	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     1.00                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      2     1.00                        bfi	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     1.00                        bfi	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     1.00                        bfc	xzr, #10, #11
+# CHECK-NEXT:  1      2     1.00                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        bfxil	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     1.00                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      2     1.00                        bfxil	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     1.00                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      2     1.00                        bfxil	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     1.00                        bfxil	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     1.00                        bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     1.00                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        lsl	x2, x3, #63
+# CHECK-NEXT:  1      2     1.00                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      2     1.00                        lsl	x9, x10, #5
+# CHECK-NEXT:  1      2     1.00                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      2     1.00                        lsl	w11, w12, #31
+# CHECK-NEXT:  1      2     1.00                        lsl	w13, w14, #29
+# CHECK-NEXT:  1      2     1.00                        ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     1.00                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     1.00                        lsr	x2, x3, #63
+# CHECK-NEXT:  1      2     1.00                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      2     1.00                        lsr	x9, x10, #5
+# CHECK-NEXT:  1      2     1.00                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      2     1.00                        lsr	w11, w12, #31
+# CHECK-NEXT:  1      2     1.00                        lsr	w13, w14, #29
+# CHECK-NEXT:  1      2     1.00                        ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      1     1.00                        cbz	w5, #4
+# CHECK-NEXT:  1      1     1.00                        cbz	x5, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	x2, #-4
+# CHECK-NEXT:  1      1     1.00                        cbnz	x26, #1048572
+# CHECK-NEXT:  1      1     1.00                        cbz	wzr, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	xzr, #0
+# CHECK-NEXT:  1      1     1.00                        b.ne	#4
+# CHECK-NEXT:  1      1     1.00                        b.ge	#1048572
+# CHECK-NEXT:  1      1     1.00                        b.ge	#-4
+# CHECK-NEXT:  1      1     1.00                        ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     1.00                        ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     1.00                        ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     1.00                        ccmp	x9, #31, #0, le
+# CHECK-NEXT:  1      1     1.00                        ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     1.00                        ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     1.00                        ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     1.00                        ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     1.00                        ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     1.00                        ccmn	x9, #31, #0, le
+# CHECK-NEXT:  1      1     1.00                        ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     1.00                        ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     1.00                        ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     1.00                        ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     1.00                        ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     1.00                        ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     1.00                        ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     1.00                        ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     1.00                        ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     1.00                        ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     1.00                        ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     1.00                        ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     1.00                        ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     1.00                        ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     1.00                        csel	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     1.00                        csel	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     1.00                        csel	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     1.00                        csel	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     1.00                        csel	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     1.00                        csel	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     1.00                        csel	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     1.00                        csel	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     1.00                        csinc	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     1.00                        csinc	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     1.00                        csinc	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     1.00                        csinc	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     1.00                        csinc	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     1.00                        csinc	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     1.00                        csinc	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     1.00                        csinc	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     1.00                        csinv	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     1.00                        csinv	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     1.00                        csinv	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     1.00                        csinv	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     1.00                        csinv	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     1.00                        csinv	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     1.00                        csinv	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     1.00                        csinv	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     1.00                        csneg	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     1.00                        csneg	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     1.00                        csneg	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     1.00                        csneg	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     1.00                        csneg	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     1.00                        csneg	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     1.00                        csneg	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     1.00                        csneg	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     1.00                        cset	w3, eq
+# CHECK-NEXT:  1      1     1.00                        cset	x9, pl
+# CHECK-NEXT:  1      1     1.00                        csetm	w20, ne
+# CHECK-NEXT:  1      1     1.00                        csetm	x30, ge
+# CHECK-NEXT:  1      1     1.00                        csinc	w2, wzr, wzr, al
+# CHECK-NEXT:  1      1     1.00                        csinv	x3, xzr, xzr, nv
+# CHECK-NEXT:  1      1     1.00                        cinc	w3, w5, gt
+# CHECK-NEXT:  1      1     1.00                        cinc	wzr, w4, le
+# CHECK-NEXT:  1      1     1.00                        cset	w9, lt
+# CHECK-NEXT:  1      1     1.00                        cinc	x3, x5, gt
+# CHECK-NEXT:  1      1     1.00                        cinc	xzr, x4, le
+# CHECK-NEXT:  1      1     1.00                        cset	x9, lt
+# CHECK-NEXT:  1      1     1.00                        csinc	w5, w6, w6, nv
+# CHECK-NEXT:  1      1     1.00                        csinc	x1, x2, x2, al
+# CHECK-NEXT:  1      1     1.00                        cinv	w3, w5, gt
+# CHECK-NEXT:  1      1     1.00                        cinv	wzr, w4, le
+# CHECK-NEXT:  1      1     1.00                        csetm	w9, lt
+# CHECK-NEXT:  1      1     1.00                        cinv	x3, x5, gt
+# CHECK-NEXT:  1      1     1.00                        cinv	xzr, x4, le
+# CHECK-NEXT:  1      1     1.00                        csetm	x9, lt
+# CHECK-NEXT:  1      1     1.00                        csinv	x1, x0, x0, al
+# CHECK-NEXT:  1      1     1.00                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      1     1.00                        cneg	w3, w5, gt
+# CHECK-NEXT:  1      1     1.00                        cneg	wzr, w4, le
+# CHECK-NEXT:  1      1     1.00                        cneg	w9, wzr, lt
+# CHECK-NEXT:  1      1     1.00                        cneg	x3, x5, gt
+# CHECK-NEXT:  1      1     1.00                        cneg	xzr, x4, le
+# CHECK-NEXT:  1      1     1.00                        cneg	x9, xzr, lt
+# CHECK-NEXT:  1      1     1.00                        csneg	x4, x8, x8, al
+# CHECK-NEXT:  1      1     1.00                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      2     1.00                        rbit	w0, w7
+# CHECK-NEXT:  1      2     1.00                        rbit	x18, x3
+# CHECK-NEXT:  1      1     1.00                        rev16	w17, w1
+# CHECK-NEXT:  1      1     1.00                        rev16	x5, x2
+# CHECK-NEXT:  1      1     1.00                        rev	w18, w0
+# CHECK-NEXT:  1      1     1.00                        rev32	x20, x1
+# CHECK-NEXT:  1      1     1.00                        rev	x22, x2
+# CHECK-NEXT:  1      1     1.00                        clz	w24, w3
+# CHECK-NEXT:  1      1     1.00                        clz	x26, x4
+# CHECK-NEXT:  1      1     1.00                        cls	w3, w5
+# CHECK-NEXT:  1      1     1.00                        cls	x20, x5
+# CHECK-NEXT:  1      12    12.00                       udiv	w0, w7, w10
+# CHECK-NEXT:  1      20    20.00                       udiv	x9, x22, x4
+# CHECK-NEXT:  1      12    12.00                       sdiv	w12, w21, w0
+# CHECK-NEXT:  1      20    20.00                       sdiv	x13, x2, x1
+# CHECK-NEXT:  1      2     1.00                        lsl	w11, w12, w13
+# CHECK-NEXT:  1      2     1.00                        lsl	x14, x15, x16
+# CHECK-NEXT:  1      2     1.00                        lsr	w17, w18, w19
+# CHECK-NEXT:  1      2     1.00                        lsr	x20, x21, x22
+# CHECK-NEXT:  1      2     1.00                        asr	w23, w24, w25
+# CHECK-NEXT:  1      2     1.00                        asr	x26, x27, x28
+# CHECK-NEXT:  1      2     1.00                        ror	w0, w1, w2
+# CHECK-NEXT:  1      2     1.00                        ror	x3, x4, x5
+# CHECK-NEXT:  1      2     1.00                        lsl	w6, w7, w8
+# CHECK-NEXT:  1      2     1.00                        lsl	x9, x10, x11
+# CHECK-NEXT:  1      2     1.00                        lsr	w12, w13, w14
+# CHECK-NEXT:  1      2     1.00                        lsr	x15, x16, x17
+# CHECK-NEXT:  1      2     1.00                        asr	w18, w19, w20
+# CHECK-NEXT:  1      2     1.00                        asr	x21, x22, x23
+# CHECK-NEXT:  1      2     1.00                        ror	w24, w25, w26
+# CHECK-NEXT:  1      2     1.00                        ror	x27, x28, x29
+# CHECK-NEXT:  1      5     2.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        smulh	xzr, x27, x26
+# CHECK-NEXT:  1      5     2.00                        umulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        umulh	x23, x30, xzr
+# CHECK-NEXT:  1      3     1.00                        madd	w1, w3, w7, w4
+# CHECK-NEXT:  1      3     1.00                        madd	wzr, w0, w9, w11
+# CHECK-NEXT:  1      3     1.00                        madd	w13, wzr, w4, w4
+# CHECK-NEXT:  1      3     1.00                        madd	w19, w30, wzr, w29
+# CHECK-NEXT:  1      3     1.00                        mul	w4, w5, w6
+# CHECK-NEXT:  1      5     2.00                        madd	x1, x3, x7, x4
+# CHECK-NEXT:  1      5     2.00                        madd	xzr, x0, x9, x11
+# CHECK-NEXT:  1      5     2.00                        madd	x13, xzr, x4, x4
+# CHECK-NEXT:  1      5     2.00                        madd	x19, x30, xzr, x29
+# CHECK-NEXT:  1      5     2.00                        mul	x4, x5, x6
+# CHECK-NEXT:  1      3     1.00                        msub	w1, w3, w7, w4
+# CHECK-NEXT:  1      3     1.00                        msub	wzr, w0, w9, w11
+# CHECK-NEXT:  1      3     1.00                        msub	w13, wzr, w4, w4
+# CHECK-NEXT:  1      3     1.00                        msub	w19, w30, wzr, w29
+# CHECK-NEXT:  1      3     1.00                        mneg	w4, w5, w6
+# CHECK-NEXT:  1      5     2.00                        msub	x1, x3, x7, x4
+# CHECK-NEXT:  1      5     2.00                        msub	xzr, x0, x9, x11
+# CHECK-NEXT:  1      5     2.00                        msub	x13, xzr, x4, x4
+# CHECK-NEXT:  1      5     2.00                        msub	x19, x30, xzr, x29
+# CHECK-NEXT:  1      5     2.00                        mneg	x4, x5, x6
+# CHECK-NEXT:  1      3     1.00                        smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        smull	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        smnegl	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        umull	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        umnegl	x19, w20, w21
+# CHECK-NEXT:  1      5     2.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        smulh	x23, x22, xzr
+# CHECK-NEXT:  1      5     2.00                        umulh	x23, x22, xzr
+# CHECK-NEXT:  1      5     2.00                        mul	x19, x20, xzr
+# CHECK-NEXT:  1      3     1.00                        mneg	w21, w22, w23
+# CHECK-NEXT:  1      3     1.00                        smull	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        umull	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        smnegl	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        umnegl	x11, w13, w17
+# CHECK-NEXT:  1      2     1.00                        extr	w3, w5, w7, #0
+# CHECK-NEXT:  1      2     1.00                        extr	w11, w13, w17, #31
+# CHECK-NEXT:  1      2     1.00                        extr	x3, x5, x7, #15
+# CHECK-NEXT:  1      2     1.00                        extr	x11, x13, x17, #63
+# CHECK-NEXT:  1      2     1.00                        ror	x19, x23, #24
+# CHECK-NEXT:  1      2     1.00                        ror	x29, xzr, #63
+# CHECK-NEXT:  1      2     1.00                        ror	w9, w13, #31
+# CHECK-NEXT:  1      3     1.00                        fcmp	s3, s5
+# CHECK-NEXT:  1      3     1.00                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmpe	s29, s30
+# CHECK-NEXT:  1      3     1.00                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmp	d4, d12
+# CHECK-NEXT:  1      3     1.00                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmpe	d26, d22
+# CHECK-NEXT:  1      3     1.00                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     1.00                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     1.00                        fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     1.00                        fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     1.00                        fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     1.00                        fccmp	d9, d31, #0, le
+# CHECK-NEXT:  1      3     1.00                        fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     1.00                        fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  1      3     1.00                        fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     1.00                        fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     1.00                        fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     1.00                        fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  1      3     1.00                        fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     1.00                        fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  1      3     1.00                        fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  1      3     1.00                        fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  1      4     1.00                        fmov	s0, s1
+# CHECK-NEXT:  1      4     1.00                        fabs	s2, s3
+# CHECK-NEXT:  1      4     1.00                        fneg	s4, s5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	s6, s7
+# CHECK-NEXT:  1      4     1.00                        fcvt	d8, s9
+# CHECK-NEXT:  1      4     1.00                        fcvt	h10, s11
+# CHECK-NEXT:  1      4     1.00                        frintn	s12, s13
+# CHECK-NEXT:  1      4     1.00                        frintp	s14, s15
+# CHECK-NEXT:  1      4     1.00                        frintm	s16, s17
+# CHECK-NEXT:  1      4     1.00                        frintz	s18, s19
+# CHECK-NEXT:  1      4     1.00                        frinta	s20, s21
+# CHECK-NEXT:  1      4     1.00                        frintx	s22, s23
+# CHECK-NEXT:  1      4     1.00                        frinti	s24, s25
+# CHECK-NEXT:  1      4     1.00                        fmov	d0, d1
+# CHECK-NEXT:  1      4     1.00                        fabs	d2, d3
+# CHECK-NEXT:  1      4     1.00                        fneg	d4, d5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	d6, d7
+# CHECK-NEXT:  1      4     1.00                        fcvt	s8, d9
+# CHECK-NEXT:  1      4     1.00                        fcvt	h10, d11
+# CHECK-NEXT:  1      4     1.00                        frintn	d12, d13
+# CHECK-NEXT:  1      4     1.00                        frintp	d14, d15
+# CHECK-NEXT:  1      4     1.00                        frintm	d16, d17
+# CHECK-NEXT:  1      4     1.00                        frintz	d18, d19
+# CHECK-NEXT:  1      4     1.00                        frinta	d20, d21
+# CHECK-NEXT:  1      4     1.00                        frintx	d22, d23
+# CHECK-NEXT:  1      4     1.00                        frinti	d24, d25
+# CHECK-NEXT:  1      4     1.00                        fcvt	s26, h27
+# CHECK-NEXT:  1      4     1.00                        fcvt	d28, h29
+# CHECK-NEXT:  1      4     1.00                        fmul	s20, s19, s17
+# CHECK-NEXT:  1      13    10.00                       fdiv	s1, s2, s3
+# CHECK-NEXT:  1      4     1.00                        fadd	s4, s5, s6
+# CHECK-NEXT:  1      4     1.00                        fsub	s7, s8, s9
+# CHECK-NEXT:  1      4     1.00                        fmax	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        fmin	s13, s14, s15
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	s16, s17, s18
+# CHECK-NEXT:  1      4     1.00                        fminnm	s19, s20, s21
+# CHECK-NEXT:  1      4     1.00                        fnmul	s22, s23, s2
+# CHECK-NEXT:  1      4     1.00                        fmul	d20, d19, d17
+# CHECK-NEXT:  1      22    19.00                       fdiv	d1, d2, d3
+# CHECK-NEXT:  1      4     1.00                        fadd	d4, d5, d6
+# CHECK-NEXT:  1      4     1.00                        fsub	d7, d8, d9
+# CHECK-NEXT:  1      4     1.00                        fmax	d10, d11, d12
+# CHECK-NEXT:  1      4     1.00                        fmin	d13, d14, d15
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	d16, d17, d18
+# CHECK-NEXT:  1      4     1.00                        fminnm	d19, d20, d21
+# CHECK-NEXT:  1      4     1.00                        fnmul	d22, d23, d24
+# CHECK-NEXT:  1      4     1.00                        fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     1.00                        fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     1.00                        fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     1.00                        fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     1.00                        fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     1.00                        fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     1.00                        fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     1.00                        fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w3, h5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w19, h0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x3, h5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, h30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x19, h0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w3, s5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w19, s0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x3, s5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, s30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x19, s0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w3, d5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w19, d0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x3, d5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, d30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x19, d0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w3, h5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w19, h0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x3, h5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x12, h30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x19, h0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w3, s5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w19, s0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x3, s5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x12, s30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x19, s0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w3, d5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w19, d0, #32
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x3, d5, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x12, d30, #45
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x19, d0, #64
+# CHECK-NEXT:  1      4     1.00                        scvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        scvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        scvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        scvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        scvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        scvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        scvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        scvtf	d14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d14, x0, #64
+# CHECK-NEXT:  1      4     1.00                        fcvtns	w3, h31
+# CHECK-NEXT:  1      4     1.00                        fcvtns	xzr, h12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	wzr, h12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	x0, h0
+# CHECK-NEXT:  1      4     1.00                        fcvtps	wzr, h9
+# CHECK-NEXT:  1      4     1.00                        fcvtps	x12, h20
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	w30, h23
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	x29, h3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	w2, h3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	x4, h5
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	w6, h7
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	x8, h9
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w10, h11
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, h13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w14, h15
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x15, h16
+# CHECK-NEXT:  1      4     1.00                        scvtf	h17, w18
+# CHECK-NEXT:  1      4     1.00                        scvtf	h19, x20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	h21, w22
+# CHECK-NEXT:  1      4     1.00                        scvtf	h23, x24
+# CHECK-NEXT:  1      4     1.00                        fcvtas	w25, h26
+# CHECK-NEXT:  1      4     1.00                        fcvtas	x27, h28
+# CHECK-NEXT:  1      4     1.00                        fcvtau	w29, h30
+# CHECK-NEXT:  1      4     1.00                        fcvtau	xzr, h0
+# CHECK-NEXT:  1      4     1.00                        fcvtns	w3, s31
+# CHECK-NEXT:  1      4     1.00                        fcvtns	xzr, s12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	wzr, s12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	x0, s0
+# CHECK-NEXT:  1      4     1.00                        fcvtps	wzr, s9
+# CHECK-NEXT:  1      4     1.00                        fcvtps	x12, s20
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	w30, s23
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	x29, s3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	w2, s3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	x4, s5
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	w6, s7
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	x8, s9
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w10, s11
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w14, s15
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x15, s16
+# CHECK-NEXT:  1      4     1.00                        scvtf	s17, w18
+# CHECK-NEXT:  1      4     1.00                        scvtf	s19, x20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s21, w22
+# CHECK-NEXT:  1      4     1.00                        scvtf	s23, x24
+# CHECK-NEXT:  1      4     1.00                        fcvtas	w25, s26
+# CHECK-NEXT:  1      4     1.00                        fcvtas	x27, s28
+# CHECK-NEXT:  1      4     1.00                        fcvtau	w29, s30
+# CHECK-NEXT:  1      4     1.00                        fcvtau	xzr, s0
+# CHECK-NEXT:  1      4     1.00                        fcvtns	w3, d31
+# CHECK-NEXT:  1      4     1.00                        fcvtns	xzr, d12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	wzr, d12
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	x0, d0
+# CHECK-NEXT:  1      4     1.00                        fcvtps	wzr, d9
+# CHECK-NEXT:  1      4     1.00                        fcvtps	x12, d20
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	w30, d23
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	x29, d3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	w2, d3
+# CHECK-NEXT:  1      4     1.00                        fcvtms	x4, d5
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	w6, d7
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	x8, d9
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	w10, d11
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	x12, d13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	w14, d15
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	x15, d16
+# CHECK-NEXT:  1      4     1.00                        scvtf	d17, w18
+# CHECK-NEXT:  1      4     1.00                        scvtf	d19, x20
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d21, w22
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d23, x24
+# CHECK-NEXT:  1      4     1.00                        fcvtas	w25, d26
+# CHECK-NEXT:  1      4     1.00                        fcvtas	x27, d28
+# CHECK-NEXT:  1      4     1.00                        fcvtau	w29, d30
+# CHECK-NEXT:  1      4     1.00                        fcvtau	xzr, d0
+# CHECK-NEXT:  1      3     1.00                        fmov	w3, s9
+# CHECK-NEXT:  1      3     1.00                        fmov	s9, w3
+# CHECK-NEXT:  1      3     1.00                        fmov	x20, d31
+# CHECK-NEXT:  1      3     1.00                        fmov	d1, x15
+# CHECK-NEXT:  1      3     1.00                        fmov	x3, v12.d[1]
+# CHECK-NEXT:  1      3     1.00                        fmov	v1.d[1], x19
+# CHECK-NEXT:  1      3     1.00                        fmov	s2, #0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	s3, #1.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	d30, #16.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	s4, #1.06250000
+# CHECK-NEXT:  1      3     1.00                        fmov	d10, #1.93750000
+# CHECK-NEXT:  1      3     1.00                        fmov	s12, #-1.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	d16, #8.50000000
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w3, #0
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x29, #4
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	xzr, #-4
+# CHECK-NEXT:  1      4     1.00    *                   ldr	s0, #8
+# CHECK-NEXT:  1      4     1.00    *                   ldr	d0, #1048572
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q0, #-1048576
+# CHECK-NEXT:  1      4     1.00                  U     prfm	pldl1strm, #0
+# CHECK-NEXT:  1      4     1.00                  U     prfm	#22, #0
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxr	w5, w6, [x17]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxr	w1, x10, [x21]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxrb	w30, [x0]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxrh	w17, [x4]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxr	w22, [sp]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxrb	w8, [x4]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxrh	w7, [x5]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxr	w6, [sp]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      4     1.00    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  2      5     2.00    *      *      U     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  2      4     2.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  1      1     1.00           *      U     stlrb	w24, [sp]
+# CHECK-NEXT:  1      1     1.00           *      U     stlrh	w25, [x30]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	w26, [x29]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      4     1.00    *             U     ldarb	w23, [sp]
+# CHECK-NEXT:  1      4     1.00    *             U     ldarh	w22, [x30]
+# CHECK-NEXT:  1      4     1.00    *             U     ldar	wzr, [x29]
+# CHECK-NEXT:  1      4     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      4     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      4     1.00    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      1     1.00           *            sturb	w9, [sp]
+# CHECK-NEXT:  1      1     1.00           *            sturh	wzr, [x12, #255]
+# CHECK-NEXT:  1      1     1.00           *            stur	w16, [x0, #-256]
+# CHECK-NEXT:  1      1     1.00           *            stur	x28, [x14, #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldurb	w1, [x20, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldurh	w20, [x1, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	w12, [sp, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	xzr, [x12, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  1      4     1.00                  U     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  1      1     1.00           *            stur	b0, [sp, #1]
+# CHECK-NEXT:  1      1     1.00           *            stur	h12, [x12, #-1]
+# CHECK-NEXT:  1      1     1.00           *            stur	s15, [x0, #255]
+# CHECK-NEXT:  1      1     1.00           *            stur	d31, [x5, #25]
+# CHECK-NEXT:  1      1     1.00           *            stur	q9, [x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	b3, [sp]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	h5, [x4, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	s7, [x12, #-1]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	d11, [x19, #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldur	q13, [x1, #2]
+# CHECK-NEXT:  2      1     1.00           *            strb	w9, [x2], #255
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3], #-256
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2], #255
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2], #1
+# CHECK-NEXT:  2      1     1.00           *            strh	w10, [x3], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	w19, [sp], #255
+# CHECK-NEXT:  2      1     1.00           *            str	w20, [x30], #1
+# CHECK-NEXT:  2      1     1.00           *            str	w21, [x12], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	xzr, [x9], #255
+# CHECK-NEXT:  2      1     1.00           *            str	x2, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            str	x19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w9, [x2], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w10, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w10, [x3], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w9, [x2], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w9, [x2], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w10, [x3], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w19, [sp], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w20, [x30], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w21, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	xzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	x2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	x19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	x2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	x2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	x2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	w2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	w2, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	b0, [x0], #255
+# CHECK-NEXT:  2      1     1.00           *            str	b3, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            str	b5, [sp], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	h10, [x10], #255
+# CHECK-NEXT:  2      1     1.00           *            str	h13, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	h15, [sp], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	s20, [x20], #255
+# CHECK-NEXT:  2      1     1.00           *            str	s23, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	s25, [x0], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	d20, [x20], #255
+# CHECK-NEXT:  2      1     1.00           *            str	d23, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	d25, [x0], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b0, [x0], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b3, [x3], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b5, [sp], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h10, [x10], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h13, [x23], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h15, [sp], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s20, [x20], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s23, [x23], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s25, [x0], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d20, [x20], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d23, [x23], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d25, [x0], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q20, [x1], #255
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q23, [x9], #1
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q25, [x20], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	q10, [x1], #255
+# CHECK-NEXT:  2      1     1.00           *            str	q22, [sp], #1
+# CHECK-NEXT:  2      1     1.00           *            str	q21, [x20], #-256
+# CHECK-NEXT:  2      4     1.00    *                   ldr	x3, [x4, #0]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w9, [x2, #255]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2, #255]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2, #1]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	w19, [sp, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	w20, [x30, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	w21, [x12, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	xzr, [x9, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	x2, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	x19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w19, [sp, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w20, [x30, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	x2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	b0, [x0, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	b3, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	b5, [sp, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	h10, [x10, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	h13, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	h15, [sp, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	s20, [x20, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	s23, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	s25, [x0, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	d20, [x20, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	d23, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	d25, [x0, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b0, [x0, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b3, [x3, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h10, [x10, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h13, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s20, [x20, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s23, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d20, [x20, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d23, [x23, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q20, [x1, #255]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q23, [x9, #1]!
+# CHECK-NEXT:  2      4     1.00    *                   ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	q10, [x1, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	q22, [sp, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	q21, [x20, #-256]!
+# CHECK-NEXT:  1      1     1.00           *            sttrb	w9, [sp]
+# CHECK-NEXT:  1      1     1.00           *            sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  1      1     1.00           *            sttr	w16, [x0, #-256]
+# CHECK-NEXT:  1      1     1.00           *            sttr	x28, [x14, #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldtr	w12, [sp, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x4, [x29]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x30, [x12, #32760]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x20, [sp, #8]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	xzr, [sp]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w2, [sp]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w17, [sp, #16380]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w13, [x2, #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w2, [x4]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w26, [x3, #121]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w12, [x2]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	xzr, [x15]
+# CHECK-NEXT:  1      1     1.00           *            str	x30, [sp]
+# CHECK-NEXT:  1      1     1.00           *            str	w20, [x4, #16380]
+# CHECK-NEXT:  1      1     1.00           *            strh	w17, [sp, #8190]
+# CHECK-NEXT:  1      1     1.00           *            strb	w23, [x3, #4095]
+# CHECK-NEXT:  1      1     1.00           *            strb	wzr, [x2]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	b31, [sp, #4095]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	h20, [x2, #8190]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	s10, [x19, #16380]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	d3, [x10, #32760]
+# CHECK-NEXT:  1      1     1.00           *            str	q12, [sp, #65520]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	s9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  1      1     1.00           *            str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x3, [sp, x5]
+# CHECK-NEXT:  1      1     1.00           *            str	x9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  1      1     1.00           *            str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  1      4     1.00                  U     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q3, [sp, x5]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q9, [x27, x6]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  1      1     1.00           *            str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      4     1.00    *                   ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	w3, w5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      4     2.00    *                   ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  2      4     2.00    *                   ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  2      4     2.00    *                   ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      1     1.00           *            stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      1     1.00           *            stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      1     1.00           *            stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      1     1.00           *            stp	q3, q5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      4     2.00    *                   ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  2      1     1.00           *            stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  3      4     2.00    *                   ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  2      1     1.00           *            stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  3      4     2.00    *                   ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  2      1     1.00           *            stp	d3, d5, [x9], #504
+# CHECK-NEXT:  2      1     1.00           *            stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  3      4     2.00    *                   ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  2      1     1.00           *            stp	q3, q5, [sp], #0
+# CHECK-NEXT:  2      1     1.00           *            stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  3      4     2.00    *                   ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  2      1     1.00           *            stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  3      4     2.00    *                   ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  2      1     1.00           *            stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  2      1     1.00           *            stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  2      1     1.00           *            stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  2      1     1.00           *            stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  2      1     1.00           *            stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  3      4     2.00    *                   ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	w3, w5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      1     1.00           *            stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      1     1.00           *            stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      1     1.00           *            stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      1     1.00           *            stnp	q3, q5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      4     2.00    *                   ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  1      1     1.00                        mov	w3, #983055
+# CHECK-NEXT:  1      1     1.00                        mov	x10, #-6148914691236517206
+# CHECK-NEXT:  1      1     1.00                        and	w12, w23, w21
+# CHECK-NEXT:  1      2     1.00                        and	w16, w15, w1, lsl #1
+# CHECK-NEXT:  1      2     1.00                        and	w9, w4, w10, lsl #31
+# CHECK-NEXT:  1      1     1.00                        and	w3, w30, w11
+# CHECK-NEXT:  1      2     1.00                        and	x3, x5, x7, lsl #63
+# CHECK-NEXT:  1      2     1.00                        and	x5, x14, x19, asr #4
+# CHECK-NEXT:  1      2     1.00                        and	w3, w17, w19, ror #31
+# CHECK-NEXT:  1      2     1.00                        and	w0, w2, wzr, lsr #17
+# CHECK-NEXT:  1      2     1.00                        and	w3, w30, w11, asr #2
+# CHECK-NEXT:  1      1     1.00                        and	xzr, x4, x26
+# CHECK-NEXT:  1      2     1.00                        and	w3, wzr, w20, ror #2
+# CHECK-NEXT:  1      2     1.00                        and	x7, x20, xzr, asr #63
+# CHECK-NEXT:  1      2     1.00                        bic	x13, x20, x14, lsl #47
+# CHECK-NEXT:  1      1     1.00                        bic	w2, w7, w9
+# CHECK-NEXT:  1      2     1.00                        orr	w2, w7, w0, asr #31
+# CHECK-NEXT:  1      2     1.00                        orr	x8, x9, x10, lsl #12
+# CHECK-NEXT:  1      2     1.00                        orn	x3, x5, x7, asr #2
+# CHECK-NEXT:  1      1     1.00                        orn	w2, w5, w29
+# CHECK-NEXT:  1      2     1.00                        ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT:  1      2     1.00                        ands	x3, x5, x20, ror #63
+# CHECK-NEXT:  1      1     1.00                        bics	w3, w5, w7
+# CHECK-NEXT:  1      2     1.00                        bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT:  1      2     1.00                        tst	w3, w7, lsl #31
+# CHECK-NEXT:  1      2     1.00                        tst	x2, x20, asr #2
+# CHECK-NEXT:  1      1     1.00                        mov	x3, x6
+# CHECK-NEXT:  1      1     1.00                        mov	x3, xzr
+# CHECK-NEXT:  1      1     1.00                        mov	wzr, w2
+# CHECK-NEXT:  1      1     1.00                        mov	w3, w5
+# CHECK-NEXT:  1      1     1.00                        movz	w2, #0, lsl #16
+# CHECK-NEXT:  1      1     1.00                        mov	w2, #-1235
+# CHECK-NEXT:  1      1     1.00                        mov	x2, #5299989643264
+# CHECK-NEXT:  1      1     1.00                        mov	x2, #0
+# CHECK-NEXT:  1      1     1.00                        movk	w3, #0
+# CHECK-NEXT:  1      1     1.00                        movz	x4, #0, lsl #16
+# CHECK-NEXT:  1      1     1.00                        movk	w5, #0, lsl #16
+# CHECK-NEXT:  1      1     1.00                        movz	x6, #0, lsl #32
+# CHECK-NEXT:  1      1     1.00                        movk	x7, #0, lsl #32
+# CHECK-NEXT:  1      1     1.00                        movz	x8, #0, lsl #48
+# CHECK-NEXT:  1      1     1.00                        movk	x9, #0, lsl #48
+# CHECK-NEXT:  1      1     1.00                        adr	x2, #1600
+# CHECK-NEXT:  1      1     1.00                        adrp	x21, #6553600
+# CHECK-NEXT:  1      1     1.00                        adr	x0, #262144
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #0
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #4
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #-32768
+# CHECK-NEXT:  1      1     1.00                        tbnz	x12, #60, #32764
+# CHECK-NEXT:  1      1     1.00                        b	#4
+# CHECK-NEXT:  1      1     1.00                        b	#-4
+# CHECK-NEXT:  1      1     1.00                        b	#134217724
+# CHECK-NEXT:  1      1     1.00                        br	x20
+# CHECK-NEXT:  1      1     1.00                        blr	xzr
+# CHECK-NEXT:  1      1     1.00                  U     ret	x10
+# CHECK-NEXT:  1      1     1.00                  U     ret
+# CHECK-NEXT:  1      1     1.00                  U     eret
+# CHECK-NEXT:  1      1     1.00                  U     drps
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA320UnitALU
+# CHECK-NEXT: [1]   - CortexA320UnitB
+# CHECK-NEXT: [2]   - CortexA320UnitDiv
+# CHECK-NEXT: [3]   - CortexA320UnitLdSt
+# CHECK-NEXT: [4]   - CortexA320UnitMAC
+# CHECK-NEXT: [5]   - CortexA320UnitPAC
+# CHECK-NEXT: [6]   - CortexA320UnitVALU
+# CHECK-NEXT: [7]   - CortexA320UnitVMAC
+# CHECK-NEXT: [8]   - CortexA320UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT: 465.00 22.00  64.00  450.00 69.00   -     221.00 12.00  87.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w2, w3, #4095
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w30, w29, #1, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w13, w5, #4095, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x5, x7, #1638
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w20, wsp, #801
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	wsp, wsp, #1104
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	wsp, w30, #4084
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x0, x24, #291
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x3, x24, #4095, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x8, sp, #1074
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	sp, x29, #3816
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w0, wsp, #4077
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w4, w20, #546, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	sp, sp, #288
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	wsp, w19, #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w13, w23, #291, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w2, #4095
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w20, wsp, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	sp, #20, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x30, #4095
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x4, sp, #3822
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w3, #291, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	wsp, #1365
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	sp, #1092, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	sp, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	wsp, w20
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x11, sp
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	w24, wsp
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w3, w5, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	wzr, w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w20, wzr, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w4, w6, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w11, w13, w15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w17, w29, w20, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w21, w22, w23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w24, w25, w26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w27, w28, w29, lsr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w2, w3, w4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w5, w6, w7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	w8, w9, w10, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x3, x5, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	xzr, x3, x5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x20, xzr, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x4, x6, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x11, x13, x15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x17, x29, x20, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x21, x22, x23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x24, x25, x26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x27, x28, x29, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x2, x3, x4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x5, x6, x7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     add	x8, x9, x10, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w20, wzr, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w4, w6, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w11, w13, w15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w2, w3, w4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w5, w6, w7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	w8, w9, w10, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x3, x5, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x3, x5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x20, xzr, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x4, x6, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x11, x13, x15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x17, x29, x20, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x2, x3, x4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x5, x6, x7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adds	x8, x9, x10, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w3, w5, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	wzr, w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w4, w6, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w11, w13, w15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w2, w3, w4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w5, w6, w7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	w8, w9, w10, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x3, x5, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	xzr, x3, x5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x4, x6, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x11, x13, x15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x17, x29, x20, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x2, x3, x4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x5, x6, x7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w3, w5, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w4, w6, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w11, w13, w15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w2, w3, w4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w5, w6, w7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	w8, w9, w10, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x3, x5, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x3, x5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x4, x6, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x11, x13, x15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x17, x29, x20, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x2, x3, x4, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x5, x6, x7, asr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     subs	x8, x9, x10, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	wzr, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w5, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w6, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w8, w9, lsl #15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w10, w11, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w12, w13, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w14, w15, lsr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w16, w17, lsr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w18, w19, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w20, w21, asr #22
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	w22, w23, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x0, x3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	xzr, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x5, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x6, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x8, x9, lsl #15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x10, x11, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x12, x13, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x14, x15, lsr #41
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x16, x17, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x18, x19, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x20, x21, asr #55
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmn	x22, x23, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w0, w3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	wzr, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w5, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w6, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w8, w9, lsl #15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w10, w11, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w12, w13, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w14, w15, lsr #21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w18, w19, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w20, w21, asr #22
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x0, x3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	xzr, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x5, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x6, x7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x8, x9, lsl #15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x10, x11, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x12, x13, lsr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x14, x15, lsr #41
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x16, x17, lsr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x18, x19, asr #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x20, x21, asr #55
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	x22, x23, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	wzr, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	w29, w27, w25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	wzr, w3, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	w9, wzr, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	w20, w0, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	x29, x27, x25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	xzr, x3, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	x9, xzr, x10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adc	x20, x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	w29, w27, w25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	wzr, w3, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	w9, wzr, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	w20, w0, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	x29, x27, x25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	xzr, x3, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	x9, xzr, x10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adcs	x20, x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	w29, w27, w25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	wzr, w3, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	w9, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	w20, w0, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	x29, x27, x25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	xzr, x3, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	x9, x10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbc	x20, x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	w29, w27, w25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	wzr, w3, w4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	w9, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	w20, w0, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	x29, x27, x25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	xzr, x3, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	x9, x10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbcs	x20, x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	w3, w12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	wzr, w9
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	w23, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	x29, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	xzr, x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngc	x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	w3, w12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	wzr, w9
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	w23, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	x29, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	xzr, x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ngcs	x0, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfx	x1, x2, #3, #2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x3, x4, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	wzr, wzr, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfx	w12, w9, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfiz	x4, x5, #52, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfx	xzr, x4, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x5, x6, #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfi	x4, x5, #52, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfc	x4, #1, #6
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	x5, x6, #12, #52
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sxtb	w1, w2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sxtb	xzr, w3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sxth	w9, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sxth	x0, w1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sxtw	x3, w30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uxtb	w1, w2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uxth	w9, w10
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfx	x3, x30, #0, #32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w3, w2, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w9, w10, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x20, x21, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w1, wzr, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w9, w10, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x20, x21, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	wzr, wzr, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w9, w10, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	x20, x21, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w1, wzr, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfiz	x2, x3, #63, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfiz	x9, x10, #5, #59
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfiz	w11, w12, #31, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfiz	w13, w14, #29, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x2, x3, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x9, x10, #5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w11, w12, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w13, w14, #29
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfi	x2, x3, #63, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfi	x9, x10, #5, #59
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfi	w11, w12, #31, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfi	w13, w14, #29, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfc	xzr, #10, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	x2, x3, #63, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	x9, x10, #5, #59
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w11, w12, #31, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	w13, w14, #29, #3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	x2, x3, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	x9, x10, #5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w11, w12, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w13, w14, #29
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x2, x3, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x9, x10, #5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w11, w12, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w13, w14, #29
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbz	w5, #4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbz	x5, #0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbnz	x2, #-4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbnz	x26, #1048572
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbz	wzr, #0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b.ne	#4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b.ge	#1048572
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b.ge	#-4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	w1, #31, #0, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	w3, #0, #15, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	wzr, #15, #13, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	x9, #31, #0, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	x3, #0, #15, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	xzr, #5, #7, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	w1, #31, #0, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	w3, #0, #15, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	wzr, #15, #13, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	x9, #31, #0, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	x3, #0, #15, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	xzr, #5, #7, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	w1, wzr, #0, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	w3, w0, #15, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	wzr, w15, #13, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	x9, xzr, #0, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	x3, x0, #15, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmp	xzr, x5, #7, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	w1, wzr, #0, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	w3, w0, #15, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	wzr, w15, #13, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	x9, xzr, #0, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	x3, x0, #15, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ccmn	xzr, x5, #7, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	w1, w0, w19, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	wzr, w5, w9, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	w9, wzr, w30, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	w1, w28, wzr, mi
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	x19, x23, x29, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	xzr, x3, x4, ge
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	x5, xzr, x6, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csel	x7, x8, xzr, lo
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	w1, w0, w19, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	wzr, w5, w9, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	w9, wzr, w30, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	w1, w28, wzr, mi
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	x19, x23, x29, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	xzr, x3, x4, ge
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	x5, xzr, x6, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	x7, x8, xzr, lo
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	w1, w0, w19, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	wzr, w5, w9, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	w9, wzr, w30, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	w1, w28, wzr, mi
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	x19, x23, x29, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	xzr, x3, x4, ge
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	x5, xzr, x6, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	x7, x8, xzr, lo
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	w1, w0, w19, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	wzr, w5, w9, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	w9, wzr, w30, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	w1, w28, wzr, mi
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	x19, x23, x29, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	xzr, x3, x4, ge
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	x5, xzr, x6, hs
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	x7, x8, xzr, lo
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cset	w3, eq
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cset	x9, pl
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csetm	w20, ne
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csetm	x30, ge
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	w2, wzr, wzr, al
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	x3, xzr, xzr, nv
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinc	w3, w5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinc	wzr, w4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cset	w9, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinc	x3, x5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinc	xzr, x4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cset	x9, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	w5, w6, w6, nv
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinc	x1, x2, x2, al
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinv	w3, w5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinv	wzr, w4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csetm	w9, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinv	x3, x5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cinv	xzr, x4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csetm	x9, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	x1, x0, x0, al
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	w3, w5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	wzr, w4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	w9, wzr, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	x3, x5, gt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	xzr, x4, le
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cneg	x9, xzr, lt
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csneg	x4, x8, x8, al
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rbit	w0, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rbit	x18, x3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rev16	w17, w1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rev16	x5, x2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rev	w18, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rev32	x20, x1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rev	x22, x2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     clz	w24, w3
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     clz	x26, x4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cls	w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cls	x20, x5
+# CHECK-NEXT:  -      -     12.00   -      -      -      -      -      -     udiv	w0, w7, w10
+# CHECK-NEXT:  -      -     20.00   -      -      -      -      -      -     udiv	x9, x22, x4
+# CHECK-NEXT:  -      -     12.00   -      -      -      -      -      -     sdiv	w12, w21, w0
+# CHECK-NEXT:  -      -     20.00   -      -      -      -      -      -     sdiv	x13, x2, x1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w11, w12, w13
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	x14, x15, x16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w17, w18, w19
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x20, x21, x22
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w23, w24, w25
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x26, x27, x28
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	w0, w1, w2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	x3, x4, x5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	w6, w7, w8
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsl	x9, x10, x11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	w12, w13, w14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     lsr	x15, x16, x17
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	w18, w19, w20
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     asr	x21, x22, x23
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	w24, w25, w26
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     smulh	xzr, x27, x26
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     umulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     umulh	x23, x30, xzr
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     madd	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     madd	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     madd	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     madd	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     mul	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     madd	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     madd	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     madd	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     madd	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     mul	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     msub	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     msub	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     msub	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     msub	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     mneg	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     msub	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     msub	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     msub	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     msub	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     mneg	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     smulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     umulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -     2.00    -      -      -      -     mul	x19, x20, xzr
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     mneg	w21, w22, w23
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     smnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -     1.00    -      -      -      -     umnegl	x11, w13, w17
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     extr	w3, w5, w7, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     extr	w11, w13, w17, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     extr	x3, x5, x7, #15
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     extr	x11, x13, x17, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	x19, x23, #24
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	x29, xzr, #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	s3, s5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	s29, s30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	d4, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	d26, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s0, s1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	s4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     29.00  fsqrt	s6, s7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	d8, s9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	s22, s23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	d0, d1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     29.00  fsqrt	d6, d7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	s8, d9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	h10, d11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	d12, d13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	d16, d17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	d18, d19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	d22, d23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	d24, d25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	s26, h27
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	d28, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     10.00  fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	s7, s8, s9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	s16, s17, s18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmul	s22, s23, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	d20, d19, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  fdiv	d1, d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	d4, d5, d6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	d7, d8, d9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	d10, d11, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	d16, d17, d18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	w3, h31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	xzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	wzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	x0, h0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	wzr, h9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	x12, h20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	w30, h23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	x29, h3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	w2, h3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	x4, h5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	w6, h7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	x8, h9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w10, h11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, h13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w14, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x15, h16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h17, w18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h19, x20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	h21, w22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	h23, x24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	w25, h26
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	x27, h28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	w29, h30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	xzr, h0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	w3, s31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	xzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	wzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	x0, s0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	wzr, s9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	x12, s20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	w30, s23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	x29, s3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	w2, s3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	x4, s5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	w6, s7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	x8, s9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w10, s11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w14, s15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x15, s16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s17, w18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s19, x20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s21, w22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s23, x24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	w25, s26
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	x27, s28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	w29, s30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	xzr, s0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	w3, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	xzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	wzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	x0, d0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	wzr, d9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	x12, d20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	w30, d23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	x29, d3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	w2, d3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	x4, d5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	w6, d7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	x8, d9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	w10, d11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	x12, d13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	w14, d15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	x15, d16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d17, w18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d19, x20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d21, w22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d23, x24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	w25, d26
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	x27, d28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	w29, d30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	w3, s9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	x20, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	d1, x15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	x3, v12.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s2, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s3, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	d30, #16.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s4, #1.06250000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	d10, #1.93750000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	s12, #-1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	d16, #8.50000000
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w3, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x29, #4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	xzr, #-4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s0, #8
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d0, #1048572
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q0, #-1048576
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     prfm	pldl1strm, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     prfm	#22, #0
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxr	w5, w6, [x17]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxr	w1, x10, [x21]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxrb	w30, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxrh	w17, [x4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxr	w22, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxrb	w8, [x4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxrh	w7, [x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxr	w6, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlrb	w24, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlrh	w25, [x30]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlr	w26, [x29]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldarb	w23, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldarh	w22, [x30]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldar	wzr, [x29]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sturb	w9, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sturh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldurb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldurh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	b0, [sp, #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	h12, [x12, #-1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	s15, [x0, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	d31, [x5, #25]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stur	q9, [x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	b3, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	h5, [x4, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	s7, [x12, #-1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	d11, [x19, #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldur	q13, [x1, #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w19, [sp], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w20, [x30], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w19, [sp], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w20, [x30], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	x2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w2, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b0, [x0], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b3, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h10, [x10], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h13, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s20, [x20], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s23, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d20, [x20], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d23, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b0, [x0], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b3, [x3], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h10, [x10], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h13, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s20, [x20], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s23, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d20, [x20], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d23, [x23], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q20, [x1], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q23, [x9], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q25, [x20], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q10, [x1], #255
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q22, [sp], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q21, [x20], #-256
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x3, [x4, #0]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q20, [x1, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q23, [x9, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q10, [x1, #255]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q22, [sp, #1]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q21, [x20, #-256]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sttrb	w9, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sttr	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     sttr	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtr	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x4, [x29]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x30, [x12, #32760]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x20, [sp, #8]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	xzr, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w2, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w17, [sp, #16380]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w13, [x2, #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w2, [x4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w26, [x3, #121]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w12, [x2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x30, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w17, [sp, #8190]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w23, [x3, #4095]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	wzr, [x2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	b31, [sp, #4095]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	h20, [x2, #8190]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s10, [x19, #16380]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d3, [x10, #32760]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	s9, [x27, x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x3, [sp, x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x9, [x27, x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q3, [sp, x5]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q9, [x27, x6]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d3, d5, [x9], #504
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q3, q5, [sp], #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	w3, #983055
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x10, #-6148914691236517206
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w12, w23, w21
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w16, w15, w1, lsl #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w9, w4, w10, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w3, w30, w11
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	x3, x5, x7, lsl #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	x5, x14, x19, asr #4
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w3, w17, w19, ror #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w0, w2, wzr, lsr #17
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w3, w30, w11, asr #2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	xzr, x4, x26
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	w3, wzr, w20, ror #2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     and	x7, x20, xzr, asr #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bic	x13, x20, x14, lsl #47
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     orr	w2, w7, w0, asr #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     orr	x8, x9, x10, lsl #12
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     orn	x3, x5, x7, asr #2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     orn	w2, w5, w29
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ands	x3, x5, x20, ror #63
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bics	w3, w5, w7
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     tst	w3, w7, lsl #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     tst	x2, x20, asr #2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x3, x6
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x3, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	wzr, w2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	w3, w5
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movz	w2, #0, lsl #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	w2, #-1235
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x2, #5299989643264
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     mov	x2, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movk	w3, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movz	x4, #0, lsl #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movk	w5, #0, lsl #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movz	x6, #0, lsl #32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movk	x7, #0, lsl #32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movz	x8, #0, lsl #48
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adr	x2, #1600
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adrp	x21, #6553600
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     adr	x0, #262144
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     tbz	x12, #62, #0
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     tbz	x12, #62, #4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b	#4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b	#-4
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     br	x20
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     blr	xzr
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     ret	x10
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     ret
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     eret
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     drps
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-neon-instructions.s
new file mode 100644
index 000000000000..147da4d2ef07
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-neon-instructions.s
@@ -0,0 +1,3208 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a320 -instruction-tables < %s | FileCheck %s
+
+abs d29, d24
+abs v0.16b, v0.16b
+abs v0.2d, v0.2d
+abs v0.2s, v0.2s
+abs v0.4h, v0.4h
+abs v0.4s, v0.4s
+abs v0.8b, v0.8b
+abs v0.8h, v0.8h
+add d17, d31, d29
+add v0.8b, v0.8b, v0.8b
+addhn v0.2s, v0.2d, v0.2d
+addhn v0.4h, v0.4s, v0.4s
+addhn v0.8b, v0.8h, v0.8h
+addhn2 v0.16b, v0.8h, v0.8h
+addhn2 v0.4s, v0.2d, v0.2d
+addhn2 v0.8h, v0.4s, v0.4s
+addp v0.2d, v0.2d, v0.2d
+addp v0.8b, v0.8b, v0.8b
+and v0.8b, v0.8b, v0.8b
+bic v0.4h, #15, lsl #8
+bic v0.8b, v0.8b, v0.8b
+bif v0.16b, v0.16b, v0.16b
+bit v0.16b, v0.16b, v0.16b
+bsl v0.8b, v0.8b, v0.8b
+cls v0.16b, v0.16b
+cls v0.2s, v0.2s
+cls v0.4h, v0.4h
+cls v0.4s, v0.4s
+cls v0.8b, v0.8b
+cls v0.8h, v0.8h
+clz v0.16b, v0.16b
+clz v0.2s, v0.2s
+clz v0.4h, v0.4h
+clz v0.4s, v0.4s
+clz v0.8b, v0.8b
+clz v0.8h, v0.8h
+cmeq d20, d21, 0
+cmeq d20, d21, d22
+cmeq v0.16b, v0.16b, 0
+cmeq v0.16b, v0.16b, v0.16b
+cmge d20, d21, 0
+cmge d20, d21, d22
+cmge v0.4h, v0.4h, v0.4h
+cmge v0.8b, v0.8b, 0
+cmgt d20, d21, 0
+cmgt d20, d21, d22
+cmgt v0.2s, v0.2s, 0
+cmgt v0.4s, v0.4s, v0.4s
+cmhi d20, d21, d22
+cmhi v0.8h, v0.8h, v0.8h
+cmhs d20, d21, d22
+cmhs v0.8b, v0.8b, v0.8b
+cmle d20, d21, 0
+cmle v0.2d, v0.2d, 0
+cmlt d20, d21, 0
+cmlt v0.8h, v0.8h, 0
+cmtst d20, d21, d22
+cmtst v0.2s, v0.2s, v0.2s
+cnt v0.16b, v0.16b
+cnt v0.8b, v0.8b
+dup v0.16b,w28
+dup v0.2d,x28
+dup v0.2s,w28
+dup v0.4h,w28
+dup v0.4s,w28
+dup v0.8b,w28
+dup v0.8h,w28
+eor v0.16b, v0.16b, v0.16b
+ext v0.16b, v0.16b, v0.16b, #3
+ext v0.8b, v0.8b, v0.8b, #3
+fabd d29, d24, d20
+fabd s29, s24, s20
+fabd v0.4s, v0.4s, v0.4s
+fabs v0.2d, v0.2d
+fabs v0.2s, v0.2s
+fabs v0.4h, v0.4h
+fabs v0.4s, v0.4s
+fabs v0.8h, v0.8h
+facge d20, d21, d22
+facge s10, s11, s12
+facge v0.4s, v0.4s, v0.4s
+facgt d20, d21, d22
+facgt s10, s11, s12
+facgt v0.2d, v0.2d, v0.2d
+fadd v0.4s, v0.4s, v0.4s
+faddp v0.2s, v0.2s, v0.2s
+faddp v0.4s, v0.4s, v0.4s
+fcmeq d20, d21, #0.0
+fcmeq d20, d21, d22
+fcmeq s10, s11, #0.0
+fcmeq s10, s11, s12
+fcmeq v0.2s, v0.2s, #0.0
+fcmeq v0.2s, v0.2s, v0.2s
+fcmge d20, d21, #0.0
+fcmge d20, d21, d22
+fcmge s10, s11, #0.0
+fcmge s10, s11, s12
+fcmge v0.2d, v0.2d, #0.0
+fcmge v0.4s, v0.4s, v0.4s
+fcmgt d20, d21, #0.0
+fcmgt d20, d21, d22
+fcmgt s10, s11, #0.0
+fcmgt s10, s11, s12
+fcmgt v0.4s, v0.4s, #0.0
+fcmgt v0.4s, v0.4s, v0.4s
+fcmle d20, d21, #0.0
+fcmle s10, s11, #0.0
+fcmle v0.2d, v0.2d, #0.0
+fcmlt d20, d21, #0.0
+fcmlt s10, s11, #0.0
+fcmlt v0.4s, v0.4s, #0.0
+fcvtas d21, d14
+fcvtas s12, s13
+fcvtas v0.2d, v0.2d
+fcvtas v0.2s, v0.2s
+fcvtas v0.4h, v0.4h
+fcvtas v0.4s, v0.4s
+fcvtas v0.8h, v0.8h
+fcvtau d21, d14
+fcvtau s12, s13
+fcvtau v0.2d, v0.2d
+fcvtau v0.2s, v0.2s
+fcvtau v0.4h, v0.4h
+fcvtau v0.4s, v0.4s
+fcvtau v0.8h, v0.8h
+fcvtl v0.2d, v0.2s
+fcvtl v0.4s, v0.4h
+fcvtl2 v0.2d, v0.4s
+fcvtl2 v0.4s, v0.8h
+fcvtms d21, d14
+fcvtms s22, s13
+fcvtms v0.2d, v0.2d
+fcvtms v0.2s, v0.2s
+fcvtms v0.4h, v0.4h
+fcvtms v0.4s, v0.4s
+fcvtms v0.8h, v0.8h
+fcvtmu d21, d14
+fcvtmu s12, s13
+fcvtmu v0.2d, v0.2d
+fcvtmu v0.2s, v0.2s
+fcvtmu v0.4h, v0.4h
+fcvtmu v0.4s, v0.4s
+fcvtmu v0.8h, v0.8h
+fcvtn v0.2s, v0.2d
+fcvtn v0.4h, v0.4s
+fcvtn2 v0.4s, v0.2d
+fcvtn2 v0.8h, v0.4s
+fcvtns d21, d14
+fcvtns s22, s13
+fcvtns v0.2d, v0.2d
+fcvtns v0.2s, v0.2s
+fcvtns v0.4h, v0.4h
+fcvtns v0.4s, v0.4s
+fcvtns v0.8h, v0.8h
+fcvtnu d21, d14
+fcvtnu s12, s13
+fcvtnu v0.2d, v0.2d
+fcvtnu v0.2s, v0.2s
+fcvtnu v0.4h, v0.4h
+fcvtnu v0.4s, v0.4s
+fcvtnu v0.8h, v0.8h
+fcvtps d21, d14
+fcvtps s22, s13
+fcvtps v0.2d, v0.2d
+fcvtps v0.2s, v0.2s
+fcvtps v0.4h, v0.4h
+fcvtps v0.4s, v0.4s
+fcvtps v0.8h, v0.8h
+fcvtpu d21, d14
+fcvtpu s12, s13
+fcvtpu v0.2d, v0.2d
+fcvtpu v0.2s, v0.2s
+fcvtpu v0.4h, v0.4h
+fcvtpu v0.4s, v0.4s
+fcvtpu v0.8h, v0.8h
+fcvtxn s22, d13
+fcvtxn v0.2s, v0.2d
+fcvtxn2 v0.4s, v0.2d
+fcvtzs d21, d12, #1
+fcvtzs d21, d14
+fcvtzs s12, s13
+fcvtzs s21, s12, #1
+fcvtzs v0.2d, v0.2d
+fcvtzs v0.2d, v0.2d, #3
+fcvtzs v0.2s, v0.2s
+fcvtzs v0.2s, v0.2s, #3
+fcvtzs v0.4h, v0.4h
+fcvtzs v0.4s, v0.4s
+fcvtzs v0.4s, v0.4s, #3
+fcvtzs v0.8h, v0.8h
+fcvtzu d21, d12, #1
+fcvtzu d21, d14
+fcvtzu s12, s13
+fcvtzu s21, s12, #1
+fcvtzu v0.2d, v0.2d
+fcvtzu v0.2d, v0.2d, #3
+fcvtzu v0.2s, v0.2s
+fcvtzu v0.2s, v0.2s, #3
+fcvtzu v0.4h, v0.4h
+fcvtzu v0.4s, v0.4s
+fcvtzu v0.4s, v0.4s, #3
+fcvtzu v0.8h, v0.8h
+fdiv v0.2s, v0.2s, v0.2s
+fmax v0.2d, v0.2d, v0.2d
+fmax v0.2s, v0.2s, v0.2s
+fmax v0.4s, v0.4s, v0.4s
+fmaxnm v0.2d, v0.2d, v0.2d
+fmaxnm v0.2s, v0.2s, v0.2s
+fmaxnm v0.4s, v0.4s, v0.4s
+fmaxnmp v0.2d, v0.2d, v0.2d
+fmaxnmp v0.2s, v0.2s, v0.2s
+fmaxnmp v0.4s, v0.4s, v0.4s
+fmaxp v0.2d, v0.2d, v0.2d
+fmaxp v0.2s, v0.2s, v0.2s
+fmaxp v0.4s, v0.4s, v0.4s
+fmin v0.2d, v0.2d, v0.2d
+fmin v0.2s, v0.2s, v0.2s
+fmin v0.4s, v0.4s, v0.4s
+fminnm v0.2d, v0.2d, v0.2d
+fminnm v0.2s, v0.2s, v0.2s
+fminnm v0.4s, v0.4s, v0.4s
+fminnmp v0.2d, v0.2d, v0.2d
+fminnmp v0.2s, v0.2s, v0.2s
+fminnmp v0.4s, v0.4s, v0.4s
+fminp v0.2d, v0.2d, v0.2d
+fminp v0.2s, v0.2s, v0.2s
+fminp v0.4s, v0.4s, v0.4s
+fmla d0, d1, v0.d[1]
+fmla s0, s1, v0.s[3]
+fmla v0.2s, v0.2s, v0.2s
+fmls d0, d4, v0.d[1]
+fmls s3, s5, v0.s[3]
+fmls v0.2s, v0.2s, v0.2s
+fmov v0.2d, #-1.25
+fmov v0.2s, #13.0
+fmov v0.4s, #1.0
+fmul d0, d1, v0.d[1]
+fmul s0, s1, v0.s[3]
+fmul v0.2s, v0.2s, v0.2s
+fmulx d0, d4, v0.d[1]
+fmulx d23, d11, d1
+fmulx s20, s22, s15
+fmulx s3, s5, v0.s[3]
+fmulx v0.2d, v0.2d, v0.2d
+fmulx v0.2s, v0.2s, v0.2s
+fmulx v0.4s, v0.4s, v0.4s
+fneg v0.2d, v0.2d
+fneg v0.2s, v0.2s
+fneg v0.4h, v0.4h
+fneg v0.4s, v0.4s
+fneg v0.8h, v0.8h
+frecpe d13, d13
+frecpe s19, s14
+frecpe v0.2d, v0.2d
+frecpe v0.2s, v0.2s
+frecpe v0.4h, v0.4h
+frecpe v0.4s, v0.4s
+frecpe v0.8h, v0.8h
+frecps  v0.4s, v0.4s, v0.4s
+frecps d22, d30, d21
+frecps s21, s16, s13
+frecpx d16, d19
+frecpx s18, s10
+frinta v0.2d, v0.2d
+frinta v0.2s, v0.2s
+frinta v0.4h, v0.4h
+frinta v0.4s, v0.4s
+frinta v0.8h, v0.8h
+frinti v0.2d, v0.2d
+frinti v0.2s, v0.2s
+frinti v0.4h, v0.4h
+frinti v0.4s, v0.4s
+frinti v0.8h, v0.8h
+frintm v0.2d, v0.2d
+frintm v0.2s, v0.2s
+frintm v0.4h, v0.4h
+frintm v0.4s, v0.4s
+frintm v0.8h, v0.8h
+frintn v0.2d, v0.2d
+frintn v0.2s, v0.2s
+frintn v0.4h, v0.4h
+frintn v0.4s, v0.4s
+frintn v0.8h, v0.8h
+frintp v0.2d, v0.2d
+frintp v0.2s, v0.2s
+frintp v0.4h, v0.4h
+frintp v0.4s, v0.4s
+frintp v0.8h, v0.8h
+frintx v0.2d, v0.2d
+frintx v0.2s, v0.2s
+frintx v0.4h, v0.4h
+frintx v0.4s, v0.4s
+frintx v0.8h, v0.8h
+frintz v0.2d, v0.2d
+frintz v0.2s, v0.2s
+frintz v0.4h, v0.4h
+frintz v0.4s, v0.4s
+frintz v0.8h, v0.8h
+frsqrte d21, d12
+frsqrte s22, s13
+frsqrte v0.2d, v0.2d
+frsqrte v0.2s, v0.2s
+frsqrte v0.4h, v0.4h
+frsqrte v0.4s, v0.4s
+frsqrte v0.8h, v0.8h
+frsqrts d8, d22, d18
+frsqrts s21, s5, s12
+frsqrts v0.2d, v0.2d, v0.2d
+fsqrt v0.2d, v0.2d
+fsqrt v0.2s, v0.2s
+fsqrt v0.4h, v0.4h
+fsqrt v0.4s, v0.4s
+fsqrt v0.8h, v0.8h
+fsub v0.2s, v0.2s, v0.2s
+ld1 { v0.16b }, [x0]
+ld1 { v0.2d, v1.2d, v2.2d }, [x0], #48
+ld1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+ld1 { v0.4s, v1.4s }, [sp], #32
+ld1 { v0.4s, v1.4s, v2.4s }, [sp]
+ld1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+ld1 { v0.8h }, [x15], x2
+ld1 { v0.8h, v1.8h }, [x15]
+ld1 { v0.b }[9], [x0]
+ld1 { v0.b }[9], [x0], #1
+ld1r { v0.16b }, [x0]
+ld1r { v0.16b }, [x0], #1
+ld1r { v0.8h }, [x15]
+ld1r { v0.8h }, [x15], #2
+ld2 { v0.16b, v1.16b }, [x0], x1
+ld2 { v0.8b, v1.8b }, [x0]
+ld2 { v0.h, v1.h }[7], [x15]
+ld2 { v0.h, v1.h }[7], [x15], #4
+ld2r { v0.2d, v1.2d }, [x0]
+ld2r { v0.2d, v1.2d }, [x0], #16
+ld2r { v0.4s, v1.4s }, [sp]
+ld2r { v0.4s, v1.4s }, [sp], #8
+ld3 { v0.4h, v1.4h, v2.4h }, [x15]
+ld3 { v0.8h, v1.8h, v2.8h }, [x15], x2
+ld3 { v0.s, v1.s, v2.s }[3], [sp]
+ld3 { v0.s, v1.s, v2.s }[3], [sp], x3
+ld3r { v0.4h, v1.4h, v2.4h }, [x15]
+ld3r { v0.4h, v1.4h, v2.4h }, [x15], #6
+ld3r { v0.8b, v1.8b, v2.8b }, [x0]
+ld3r { v0.8b, v1.8b, v2.8b }, [x0], #3
+ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+ld4 { v0.d, v1.d, v2.d, v3.d }[1], [x0]
+ld4 { v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+ld4 { v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+ld4r { v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+ld4r { v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+mla v0.8b, v0.8b, v0.8b
+mls v0.4h, v0.4h, v0.4h
+mov b0, v0.b[15]
+mov d6, v0.d[1]
+mov h2, v0.h[5]
+mov s17, v0.s[2]
+mov v0.16b, v0.16b
+mov v0.8b, v0.8b
+movi d15, #0xff00ff00ff00ff
+movi v0.16b, #31
+movi v0.2d, #0xff0000ff0000ffff
+movi v0.2s, #8, msl #8
+movi v0.4s, #255, lsl #24
+movi v0.8b, #255
+mul v0.8b, v0.8b, v0.8b
+mvni v0.2s, 0
+mvni v0.4s, #16, msl #16
+neg d29, d24
+neg v0.16b, v0.16b
+neg v0.2d, v0.2d
+neg v0.2s, v0.2s
+neg v0.4h, v0.4h
+neg v0.4s, v0.4s
+neg v0.8b, v0.8b
+neg v0.8h, v0.8h
+not v0.16b, v0.16b
+not v0.8b, v0.8b
+orn v0.16b, v0.16b, v0.16b
+orr v0.16b, v0.16b, v0.16b
+orr v0.8h, #31
+pmul v0.16b, v0.16b, v0.16b
+pmul v0.8b, v0.8b, v0.8b
+pmull v0.8h, v0.8b, v0.8b
+pmull2 v0.8h, v0.16b, v0.16b
+raddhn v0.2s, v0.2d, v0.2d
+raddhn v0.4h, v0.4s, v0.4s
+raddhn v0.8b, v0.8h, v0.8h
+raddhn2 v0.16b, v0.8h, v0.8h
+raddhn2 v0.4s, v0.2d, v0.2d
+raddhn2 v0.8h, v0.4s, v0.4s
+rbit v0.16b, v0.16b
+rbit v0.8b, v0.8b
+rev16 v21.8b, v1.8b
+rev16 v30.16b, v31.16b
+rev32 v0.4h, v9.4h
+rev32 v21.8b, v1.8b
+rev32 v30.16b, v31.16b
+rev32 v4.8h, v7.8h
+rev64 v0.16b, v31.16b
+rev64 v1.8b, v9.8b
+rev64 v13.4h, v21.4h
+rev64 v2.8h, v4.8h
+rev64 v4.2s, v0.2s
+rev64 v6.4s, v8.4s
+rshrn v0.2s, v0.2d, #3
+rshrn v0.4h, v0.4s, #3
+rshrn v0.8b, v0.8h, #3
+rshrn2 v0.16b, v0.8h, #3
+rshrn2 v0.4s, v0.2d, #3
+rshrn2 v0.8h, v0.4s, #3
+rsubhn v0.2s, v0.2d, v0.2d
+rsubhn v0.4h, v0.4s, v0.4s
+rsubhn v0.8b, v0.8h, v0.8h
+rsubhn2 v0.16b, v0.8h, v0.8h
+rsubhn2 v0.4s, v0.2d, v0.2d
+rsubhn2 v0.8h, v0.4s, v0.4s
+saba v0.16b, v0.16b, v0.16b
+sabal v0.2d, v0.2s, v0.2s
+sabal v0.4s, v0.4h, v0.4h
+sabal v0.8h, v0.8b, v0.8b
+sabal2 v0.2d, v0.4s, v0.4s
+sabal2 v0.4s, v0.8h, v0.8h
+sabal2 v0.8h, v0.16b, v0.16b
+sabd v0.4h, v0.4h, v0.4h
+sabdl v0.2d, v0.2s, v0.2s
+sabdl v0.4s, v0.4h, v0.4h
+sabdl v0.8h, v0.8b, v0.8b
+sabdl2 v0.2d, v0.4s, v0.4s
+sabdl2 v0.4s, v0.8h, v0.8h
+sabdl2 v0.8h, v0.16b, v0.16b
+sadalp v0.1d, v0.2s
+sadalp v0.2d, v0.4s
+sadalp v0.2s, v0.4h
+sadalp v0.4h, v0.8b
+sadalp v0.4s, v0.8h
+sadalp v0.8h, v0.16b
+saddl v0.2d, v0.2s, v0.2s
+saddl v0.4s, v0.4h, v0.4h
+saddl v0.8h, v0.8b, v0.8b
+saddl2 v0.2d, v0.4s, v0.4s
+saddl2 v0.4s, v0.8h, v0.8h
+saddl2 v0.8h, v0.16b, v0.16b
+saddlp v0.1d, v0.2s
+saddlp v0.2d, v0.4s
+saddlp v0.2s, v0.4h
+saddlp v0.4h, v0.8b
+saddlp v0.4s, v0.8h
+saddlp v0.8h, v0.16b
+saddw v0.2d, v0.2d, v0.2s
+saddw v0.4s, v0.4s, v0.4h
+saddw v0.8h, v0.8h, v0.8b
+saddw2 v0.2d, v0.2d, v0.4s
+saddw2 v0.4s, v0.4s, v0.8h
+saddw2 v0.8h, v0.8h, v0.16b
+scvtf d21, d12
+scvtf d21, d12, #64
+scvtf s22, s13
+scvtf s22, s13, #32
+scvtf v0.2d, v0.2d
+scvtf v0.2d, v0.2d, #3
+scvtf v0.2s, v0.2s
+scvtf v0.2s, v0.2s, #3
+scvtf v0.4h, v0.4h
+scvtf v0.4s, v0.4s
+scvtf v0.4s, v0.4s, #3
+scvtf v0.8h, v0.8h
+shadd v0.8b, v0.8b, v0.8b
+shl d7, d10, #12
+shl v0.16b, v0.16b, #3
+shl v0.2d, v0.2d, #3
+shl v0.4h, v0.4h, #3
+shl v0.4s, v0.4s, #3
+shll	v0.2d, v0.2s, #32
+shll	v0.4s, v0.4h, #16
+shll	v0.8h, v0.8b, #8
+shll v0.2d, v0.2s, #32
+shll v0.4s, v0.4h, #16
+shll v0.8h, v0.8b, #8
+shll2	v0.2d, v0.4s, #32
+shll2	v0.4s, v0.8h, #16
+shll2	v0.8h, v0.16b, #8
+shll2 v0.2d, v0.4s, #32
+shll2 v0.4s, v0.8h, #16
+shll2 v0.8h, v0.16b, #8
+shrn v0.2s, v0.2d, #3
+shrn v0.4h, v0.4s, #3
+shrn v0.8b, v0.8h, #3
+shrn2 v0.16b, v0.8h, #3
+shrn2 v0.4s, v0.2d, #3
+shrn2 v0.8h, v0.4s, #3
+shsub v0.2s, v0.2s, v0.2s
+shsub v0.4h, v0.4h, v0.4h
+sli d10, d14, #12
+sli v0.16b, v0.16b, #3
+sli v0.2d, v0.2d, #3
+sli v0.2s, v0.2s, #3
+sli v0.4h, v0.4h, #3
+sli v0.4s, v0.4s, #3
+sli v0.8b, v0.8b, #3
+sli v0.8h, v0.8h, #3
+smax v0.2s, v0.2s, v0.2s
+smax v0.4h, v0.4h, v0.4h
+smax v0.8b, v0.8b, v0.8b
+smaxp v0.2s, v0.2s, v0.2s
+smaxp v0.4h, v0.4h, v0.4h
+smaxp v0.8b, v0.8b, v0.8b
+smin v0.16b, v0.16b, v0.16b
+smin v0.4s, v0.4s, v0.4s
+smin v0.8h, v0.8h, v0.8h
+sminp v0.16b, v0.16b, v0.16b
+sminp v0.4s, v0.4s, v0.4s
+sminp v0.8h, v0.8h, v0.8h
+smlal v0.2d, v0.2s, v0.2s
+smlal v0.4s, v0.4h, v0.4h
+smlal v0.8h, v0.8b, v0.8b
+smlal2 v0.2d, v0.4s, v0.4s
+smlal2 v0.4s, v0.8h, v0.8h
+smlal2 v0.8h, v0.16b, v0.16b
+smlsl v0.2d, v0.2s, v0.2s
+smlsl v0.4s, v0.4h, v0.4h
+smlsl v0.8h, v0.8b, v0.8b
+smlsl2 v0.2d, v0.4s, v0.4s
+smlsl2 v0.4s, v0.8h, v0.8h
+smlsl2 v0.8h, v0.16b, v0.16b
+smull v0.2d, v0.2s, v0.2s
+smull v0.4s, v0.4h, v0.4h
+smull v0.8h, v0.8b, v0.8b
+smull2 v0.2d, v0.4s, v0.4s
+smull2 v0.4s, v0.8h, v0.8h
+smull2 v0.8h, v0.16b, v0.16b
+sqabs b19, b14
+sqabs d18, d12
+sqabs h21, h15
+sqabs s20, s12
+sqabs v0.16b, v0.16b
+sqabs v0.2d, v0.2d
+sqabs v0.2s, v0.2s
+sqabs v0.4h, v0.4h
+sqabs v0.4s, v0.4s
+sqabs v0.8b, v0.8b
+sqabs v0.8h, v0.8h
+sqadd b20, b11, b15
+sqadd v0.16b, v0.16b, v0.16b
+sqadd v0.2s, v0.2s, v0.2s
+sqdmlal d19, s24, s12
+sqdmlal d8, s9, v0.s[1]
+sqdmlal s0, h0, v0.h[3]
+sqdmlal s17, h27, h12
+sqdmlal v0.2d, v0.2s, v0.2s
+sqdmlal v0.4s, v0.4h, v0.4h
+sqdmlal2 v0.2d, v0.4s, v0.4s
+sqdmlal2 v0.4s, v0.8h, v0.8h
+sqdmlsl d12, s23, s13
+sqdmlsl d8, s9, v0.s[1]
+sqdmlsl s0, h0, v0.h[3]
+sqdmlsl s14, h12, h25
+sqdmlsl v0.2d, v0.2s, v0.2s
+sqdmlsl v0.4s, v0.4h, v0.4h
+sqdmlsl2 v0.2d, v0.4s, v0.4s
+sqdmlsl2 v0.4s, v0.8h, v0.8h
+sqdmulh h10, h11, h12
+sqdmulh h7, h15, v0.h[3]
+sqdmulh s15, s14, v0.s[1]
+sqdmulh s20, s21, s2
+sqdmulh v0.2s, v0.2s, v0.2s
+sqdmulh v0.4s, v0.4s, v0.4s
+sqdmull d1, s1, v0.s[1]
+sqdmull d15, s22, s12
+sqdmull s1, h1, v0.h[3]
+sqdmull s12, h22, h12
+sqdmull v0.2d, v0.2s, v0.2s
+sqdmull v0.4s, v0.4h, v0.4h
+sqdmull2 v0.2d, v0.4s, v0.4s
+sqdmull2 v0.4s, v0.8h, v0.8h
+sqneg b19, b14
+sqneg d18, d12
+sqneg h21, h15
+sqneg s20, s12
+sqneg v0.16b, v0.16b
+sqneg v0.2d, v0.2d
+sqneg v0.2s, v0.2s
+sqneg v0.4h, v0.4h
+sqneg v0.4s, v0.4s
+sqneg v0.8b, v0.8b
+sqneg v0.8h, v0.8h
+sqrdmulh h10, h11, h12
+sqrdmulh h7, h15, v0.h[3]
+sqrdmulh s15, s14, v0.s[1]
+sqrdmulh s20, s21, s2
+sqrdmulh v0.4h, v0.4h, v0.4h
+sqrdmulh v0.8h, v0.8h, v0.8h
+sqrshl d31, d31, d31
+sqrshl h3, h4, h15
+sqrshl v0.2s, v0.2s, v0.2s
+sqrshl v0.4h, v0.4h, v0.4h
+sqrshl v0.8b, v0.8b, v0.8b
+sqrshrn b10, h13, #2
+sqrshrn h15, s10, #6
+sqrshrn s15, d12, #9
+sqrshrn v0.2s, v0.2d, #3
+sqrshrn v0.4h, v0.4s, #3
+sqrshrn v0.8b, v0.8h, #3
+sqrshrn2 v0.16b, v0.8h, #3
+sqrshrn2 v0.4s, v0.2d, #3
+sqrshrn2 v0.8h, v0.4s, #3
+sqrshrun b17, h10, #6
+sqrshrun h10, s13, #15
+sqrshrun s22, d16, #31
+sqrshrun v0.2s, v0.2d, #3
+sqrshrun v0.4h, v0.4s, #3
+sqrshrun v0.8b, v0.8h, #3
+sqrshrun2 v0.16b, v0.8h, #3
+sqrshrun2 v0.4s, v0.2d, #3
+sqrshrun2 v0.8h, v0.4s, #3
+sqshl b11, b19, #7
+sqshl d15, d16, #51
+sqshl d31, d31, d31
+sqshl h13, h18, #11
+sqshl h3, h4, h15
+sqshl s14, s17, #22
+sqshl v0.16b, v0.16b, #3
+sqshl v0.2d, v0.2d, #3
+sqshl v0.2s, v0.2s, #3
+sqshl v0.2s, v0.2s, v0.2s
+sqshl v0.4h, v0.4h, #3
+sqshl v0.4h, v0.4h, v0.4h
+sqshl v0.4s, v0.4s, #3
+sqshl v0.8b, v0.8b, #3
+sqshl v0.8b, v0.8b, v0.8b
+sqshl v0.8h, v0.8h, #3
+sqshlu b15, b18, #6
+sqshlu d11, d13, #32
+sqshlu h19, h17, #6
+sqshlu s16, s14, #25
+sqshlu v0.16b, v0.16b, #3
+sqshlu v0.2d, v0.2d, #3
+sqshlu v0.2s, v0.2s, #3
+sqshlu v0.4h, v0.4h, #3
+sqshlu v0.4s, v0.4s, #3
+sqshlu v0.8b, v0.8b, #3
+sqshlu v0.8h, v0.8h, #3
+sqshrn b10, h15, #5
+sqshrn h17, s10, #4
+sqshrn s18, d10, #31
+sqshrn v0.2s, v0.2d, #3
+sqshrn v0.4h, v0.4s, #3
+sqshrn v0.8b, v0.8h, #3
+sqshrn2 v0.16b, v0.8h, #3
+sqshrn2 v0.4s, v0.2d, #3
+sqshrn2 v0.8h, v0.4s, #3
+sqshrun b15, h10, #7
+sqshrun h20, s14, #3
+sqshrun s10, d15, #15
+sqshrun v0.2s, v0.2d, #3
+sqshrun v0.4h, v0.4s, #3
+sqshrun v0.8b, v0.8h, #3
+sqshrun2 v0.16b, v0.8h, #3
+sqshrun2 v0.4s, v0.2d, #3
+sqshrun2 v0.8h, v0.4s, #3
+sqsub s20, s10, s7
+sqsub v0.2d, v0.2d, v0.2d
+sqsub v0.4s, v0.4s, v0.4s
+sqsub v0.8b, v0.8b, v0.8b
+sqxtn b18, h18
+sqxtn h20, s17
+sqxtn s19, d14
+sqxtn v0.2s, v0.2d
+sqxtn v0.4h, v0.4s
+sqxtn v0.8b, v0.8h
+sqxtn2 v0.16b, v0.8h
+sqxtn2 v0.4s, v0.2d
+sqxtn2 v0.8h, v0.4s
+sqxtun b19, h14
+sqxtun h21, s15
+sqxtun s20, d12
+sqxtun v0.2s, v0.2d
+sqxtun v0.4h, v0.4s
+sqxtun v0.8b, v0.8h
+sqxtun2 v0.16b, v0.8h
+sqxtun2 v0.4s, v0.2d
+sqxtun2 v0.8h, v0.4s
+srhadd v0.2s, v0.2s, v0.2s
+srhadd v0.4h, v0.4h, v0.4h
+srhadd v0.8b, v0.8b, v0.8b
+sri d10, d12, #14
+sri v0.16b, v0.16b, #3
+sri v0.2d, v0.2d, #3
+sri v0.2s, v0.2s, #3
+sri v0.4h, v0.4h, #3
+sri v0.4s, v0.4s, #3
+sri v0.8b, v0.8b, #3
+sri v0.8h, v0.8h, #3
+srshl d16, d16, d16
+srshl v0.2s, v0.2s, v0.2s
+srshl v0.4h, v0.4h, v0.4h
+srshl v0.8b, v0.8b, v0.8b
+srshr d19, d18, #7
+srshr v0.16b, v0.16b, #3
+srshr v0.2d, v0.2d, #3
+srshr v0.2s, v0.2s, #3
+srshr v0.4h, v0.4h, #3
+srshr v0.4s, v0.4s, #3
+srshr v0.8b, v0.8b, #3
+srshr v0.8h, v0.8h, #3
+srsra d15, d11, #19
+srsra v0.16b, v0.16b, #3
+srsra v0.2d, v0.2d, #3
+srsra v0.2s, v0.2s, #3
+srsra v0.4h, v0.4h, #3
+srsra v0.4s, v0.4s, #3
+srsra v0.8b, v0.8b, #3
+srsra v0.8h, v0.8h, #3
+sshl d31, d31, d31
+sshl v0.2d, v0.2d, v0.2d
+sshl v0.2s, v0.2s, v0.2s
+sshl v0.4h, v0.4h, v0.4h
+sshl v0.8b, v0.8b, v0.8b
+sshll v0.2d, v0.2s, #3
+sshll2 v0.4s, v0.8h, #3
+sshr d15, d16, #12
+sshr v0.16b, v0.16b, #3
+sshr v0.2d, v0.2d, #3
+sshr v0.2s, v0.2s, #3
+sshr v0.4h, v0.4h, #3
+sshr v0.4s, v0.4s, #3
+sshr v0.8b, v0.8b, #3
+sshr v0.8h, v0.8h, #3
+ssra d18, d12, #21
+ssra v0.16b, v0.16b, #3
+ssra v0.2d, v0.2d, #3
+ssra v0.2s, v0.2s, #3
+ssra v0.4h, v0.4h, #3
+ssra v0.4s, v0.4s, #3
+ssra v0.8b, v0.8b, #3
+ssra v0.8h, v0.8h, #3
+ssubl v0.2d, v0.2s, v0.2s
+ssubl v0.4s, v0.4h, v0.4h
+ssubl v0.8h, v0.8b, v0.8b
+ssubl2 v0.2d, v0.4s, v0.4s
+ssubl2 v0.4s, v0.8h, v0.8h
+ssubl2 v0.8h, v0.16b, v0.16b
+ssubw v0.2d, v0.2d, v0.2s
+ssubw v0.4s, v0.4s, v0.4h
+ssubw v0.8h, v0.8h, v0.8b
+ssubw2 v0.2d, v0.2d, v0.4s
+ssubw2 v0.4s, v0.4s, v0.8h
+ssubw2 v0.8h, v0.8h, v0.16b
+st1 { v0.16b }, [x0]
+st1 { v0.2d, v1.2d, v2.2d }, [x0], #48
+st1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+st1 { v0.4s, v1.4s }, [sp], #32
+st1 { v0.4s, v1.4s, v2.4s }, [sp]
+st1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+st1 { v0.8h }, [x15], x2
+st1 { v0.8h, v1.8h }, [x15]
+st1 { v0.d }[1], [x0]
+st1 { v0.d }[1], [x0], #8
+st2 { v0.16b, v1.16b }, [x0], x1
+st2 { v0.8b, v1.8b }, [x0]
+st2 { v0.s, v1.s }[3], [sp]
+st2 { v0.s, v1.s }[3], [sp], #8
+st3 { v0.4h, v1.4h, v2.4h }, [x15]
+st3 { v0.8h, v1.8h, v2.8h }, [x15], x2
+st3 { v0.h, v1.h, v2.h }[7], [x15]
+st3 { v0.h, v1.h, v2.h }[7], [x15], #6
+st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0]
+st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+sub d15, d5, d16
+sub v0.2d, v0.2d, v0.2d
+suqadd b19, b14
+suqadd d18, d22
+suqadd h20, h15
+suqadd s21, s12
+suqadd v0.16b, v0.16b
+suqadd v0.2d, v0.2d
+suqadd v0.2s, v0.2s
+suqadd v0.4h, v0.4h
+suqadd v0.4s, v0.4s
+suqadd v0.8b, v0.8b
+suqadd v0.8h, v0.8h
+tbl v0.16b, { v0.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+tbl v0.8b, { v0.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+tbx v0.16b, { v0.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+tbx v0.8b, { v0.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+trn1	v0.16b, v0.16b, v0.16b
+trn1	v0.2d, v0.2d, v0.2d
+trn1	v0.2s, v0.2s, v0.2s
+trn1	v0.4h, v0.4h, v0.4h
+trn1	v0.4s, v0.4s, v0.4s
+trn1	v0.8b, v0.8b, v0.8b
+trn1	v0.8h, v0.8h, v0.8h
+trn2	v0.16b, v0.16b, v0.16b
+trn2	v0.2d, v0.2d, v0.2d
+trn2	v0.2s, v0.2s, v0.2s
+trn2	v0.4h, v0.4h, v0.4h
+trn2	v0.4s, v0.4s, v0.4s
+trn2	v0.8b, v0.8b, v0.8b
+trn2	v0.8h, v0.8h, v0.8h
+uaba v0.8b, v0.8b, v0.8b
+uabal v0.2d, v0.2s, v0.2s
+uabal v0.4s, v0.4h, v0.4h
+uabal v0.8h, v0.8b, v0.8b
+uabal2 v0.2d, v0.4s, v0.4s
+uabal2 v0.4s, v0.8h, v0.8h
+uabal2 v0.8h, v0.16b, v0.16b
+uabd v0.4h, v0.4h, v0.4h
+uabdl v0.2d, v0.2s, v0.2s
+uabdl v0.4s, v0.4h, v0.4h
+uabdl v0.8h, v0.8b, v0.8b
+uabdl2 v0.2d, v0.4s, v0.4s
+uabdl2 v0.4s, v0.8h, v0.8h
+uabdl2 v0.8h, v0.16b, v0.16b
+uadalp v0.1d, v0.2s
+uadalp v0.2d, v0.4s
+uadalp v0.2s, v0.4h
+uadalp v0.4h, v0.8b
+uadalp v0.4s, v0.8h
+uadalp v0.8h, v0.16b
+uaddl v0.2d, v0.2s, v0.2s
+uaddl v0.4s, v0.4h, v0.4h
+uaddl v0.8h, v0.8b, v0.8b
+uaddl2 v0.2d, v0.4s, v0.4s
+uaddl2 v0.4s, v0.8h, v0.8h
+uaddl2 v0.8h, v0.16b, v0.16b
+uaddlp v0.1d, v0.2s
+uaddlp v0.2d, v0.4s
+uaddlp v0.2s, v0.4h
+uaddlp v0.4h, v0.8b
+uaddlp v0.4s, v0.8h
+uaddlp v0.8h, v0.16b
+uaddw v0.2d, v0.2d, v0.2s
+uaddw v0.4s, v0.4s, v0.4h
+uaddw v0.8h, v0.8h, v0.8b
+uaddw2 v0.2d, v0.2d, v0.4s
+uaddw2 v0.4s, v0.4s, v0.8h
+uaddw2 v0.8h, v0.8h, v0.16b
+ucvtf d21, d14
+ucvtf d21, d14, #64
+ucvtf s22, s13
+ucvtf s22, s13, #32
+ucvtf v0.2d, v0.2d
+ucvtf v0.2d, v0.2d, #3
+ucvtf v0.2s, v0.2s
+ucvtf v0.2s, v0.2s, #3
+ucvtf v0.4h, v0.4h
+ucvtf v0.4s, v0.4s
+ucvtf v0.4s, v0.4s, #3
+ucvtf v0.8h, v0.8h
+uhadd v0.16b, v0.16b, v0.16b
+uhadd v0.8h, v0.8h, v0.8h
+uhsub v0.4s, v0.4s, v0.4s
+umax v0.16b, v0.16b, v0.16b
+umax v0.4s, v0.4s, v0.4s
+umax v0.8h, v0.8h, v0.8h
+umaxp v0.16b, v0.16b, v0.16b
+umaxp v0.4s, v0.4s, v0.4s
+umaxp v0.8h, v0.8h, v0.8h
+umin v0.2s, v0.2s, v0.2s
+umin v0.4h, v0.4h, v0.4h
+umin v0.8b, v0.8b, v0.8b
+uminp v0.2s, v0.2s, v0.2s
+uminp v0.4h, v0.4h, v0.4h
+uminp v0.8b, v0.8b, v0.8b
+umlal v0.2d, v0.2s, v0.2s
+umlal v0.4s, v0.4h, v0.4h
+umlal v0.8h, v0.8b, v0.8b
+umlal2 v0.2d, v0.4s, v0.4s
+umlal2 v0.4s, v0.8h, v0.8h
+umlal2 v0.8h, v0.16b, v0.16b
+umlsl v0.2d, v0.2s, v0.2s
+umlsl v0.4s, v0.4h, v0.4h
+umlsl v0.8h, v0.8b, v0.8b
+umlsl2 v0.2d, v0.4s, v0.4s
+umlsl2 v0.4s, v0.8h, v0.8h
+umlsl2 v0.8h, v0.16b, v0.16b
+umull v0.2d, v0.2s, v0.2s
+umull v0.4s, v0.4h, v0.4h
+umull v0.8h, v0.8b, v0.8b
+umull2 v0.2d, v0.4s, v0.4s
+umull2 v0.4s, v0.8h, v0.8h
+umull2 v0.8h, v0.16b, v0.16b
+uqadd h0, h1, h5
+uqadd v0.8h, v0.8h, v0.8h
+uqrshl b11, b20, b30
+uqrshl s23, s20, s16
+uqrshl v0.16b, v0.16b, v0.16b
+uqrshl v0.4s, v0.4s, v0.4s
+uqrshl v0.4s, v0.4s, v0.4s
+uqrshl v0.8h, v0.8h, v0.8h
+uqrshrn b10, h12, #5
+uqrshrn h12, s10, #14
+uqrshrn s10, d10, #25
+uqrshrn v0.2s, v0.2d, #3
+uqrshrn v0.4h, v0.4s, #3
+uqrshrn v0.8b, v0.8h, #3
+uqrshrn2 v0.16b, v0.8h, #3
+uqrshrn2 v0.4s, v0.2d, #3
+uqrshrn2 v0.8h, v0.4s, #3
+uqshl b11, b20, b30
+uqshl b18, b15, #6
+uqshl d15, d12, #19
+uqshl h11, h18, #7
+uqshl s14, s19, #18
+uqshl s23, s20, s16
+uqshl v0.16b, v0.16b, #3
+uqshl v0.16b, v0.16b, v0.16b
+uqshl v0.2d, v0.2d, #3
+uqshl v0.2d, v0.2d, v0.2d
+uqshl v0.2s, v0.2s, #3
+uqshl v0.4h, v0.4h, #3
+uqshl v0.4s, v0.4s, #3
+uqshl v0.4s, v0.4s, v0.4s
+uqshl v0.8b, v0.8b, #3
+uqshl v0.8h, v0.8h, #3
+uqshl v0.8h, v0.8h, v0.8h
+uqshrn b12, h10, #7
+uqshrn h10, s14, #5
+uqshrn s10, d12, #13
+uqshrn v0.2s, v0.2d, #3
+uqshrn v0.4h, v0.4s, #3
+uqshrn v0.8b, v0.8h, #3
+uqshrn2 v0.16b, v0.8h, #3
+uqshrn2 v0.4s, v0.2d, #3
+uqshrn2 v0.8h, v0.4s, #3
+uqsub d16, d16, d16
+uqsub v0.4h, v0.4h, v0.4h
+uqxtn b18, h18
+uqxtn h20, s17
+uqxtn s19, d14
+uqxtn v0.2s, v0.2d
+uqxtn v0.4h, v0.4s
+uqxtn v0.8b, v0.8h
+uqxtn2 v0.16b, v0.8h
+uqxtn2 v0.4s, v0.2d
+uqxtn2 v0.8h, v0.4s
+urecpe v0.2s, v0.2s
+urecpe v0.4s, v0.4s
+urhadd v0.16b, v0.16b, v0.16b
+urhadd v0.4s, v0.4s, v0.4s
+urhadd v0.8h, v0.8h, v0.8h
+urshl d8, d7, d4
+urshl v0.16b, v0.16b, v0.16b
+urshl v0.2d, v0.2d, v0.2d
+urshl v0.4s, v0.4s, v0.4s
+urshl v0.8h, v0.8h, v0.8h
+urshr d20, d23, #31
+urshr v0.16b, v0.16b, #3
+urshr v0.2d, v0.2d, #3
+urshr v0.2s, v0.2s, #3
+urshr v0.4h, v0.4h, #3
+urshr v0.4s, v0.4s, #3
+urshr v0.8b, v0.8b, #3
+urshr v0.8h, v0.8h, #3
+ursqrte v0.2s, v0.2s
+ursqrte v0.4s, v0.4s
+ursra d18, d10, #13
+ursra v0.16b, v0.16b, #3
+ursra v0.2d, v0.2d, #3
+ursra v0.2s, v0.2s, #3
+ursra v0.4h, v0.4h, #3
+ursra v0.4s, v0.4s, #3
+ursra v0.8b, v0.8b, #3
+ursra v0.8h, v0.8h, #3
+ushl d0, d0, d0
+ushl v0.16b, v0.16b, v0.16b
+ushl v0.4s, v0.4s, v0.4s
+ushl v0.8h, v0.8h, v0.8h
+ushll v0.4s, v0.4h, #3
+ushll2 v0.8h, v0.16b, #3
+ushr d10, d17, #18
+ushr v0.16b, v0.16b, #3
+ushr v0.2d, v0.2d, #3
+ushr v0.2s, v0.2s, #3
+ushr v0.4h, v0.4h, #3
+ushr v0.4s, v0.4s, #3
+ushr v0.8b, v0.8b, #3
+ushr v0.8h, v0.8h, #3
+usqadd b19, b14
+usqadd d18, d22
+usqadd h20, h15
+usqadd s21, s12
+usqadd v0.16b, v0.16b
+usqadd v0.2d, v0.2d
+usqadd v0.2s, v0.2s
+usqadd v0.4h, v0.4h
+usqadd v0.4s, v0.4s
+usqadd v0.8b, v0.8b
+usqadd v0.8h, v0.8h
+usra d20, d13, #61
+usra v0.16b, v0.16b, #3
+usra v0.2d, v0.2d, #3
+usra v0.2s, v0.2s, #3
+usra v0.4h, v0.4h, #3
+usra v0.4s, v0.4s, #3
+usra v0.8b, v0.8b, #3
+usra v0.8h, v0.8h, #3
+usubl v0.2d, v0.2s, v0.2s
+usubl v0.4s, v0.4h, v0.4h
+usubl v0.8h, v0.8b, v0.8b
+usubl2 v0.2d, v0.4s, v0.4s
+usubl2 v0.4s, v0.8h, v0.8h
+usubl2 v0.8h, v0.16b, v0.16b
+usubw v0.2d, v0.2d, v0.2s
+usubw v0.4s, v0.4s, v0.4h
+usubw v0.8h, v0.8h, v0.8b
+usubw2 v0.2d, v0.2d, v0.4s
+usubw2 v0.4s, v0.4s, v0.8h
+usubw2 v0.8h, v0.8h, v0.16b
+uzp1	v0.16b, v0.16b, v0.16b
+uzp1	v0.2d, v0.2d, v0.2d
+uzp1	v0.2s, v0.2s, v0.2s
+uzp1	v0.4h, v0.4h, v0.4h
+uzp1	v0.4s, v0.4s, v0.4s
+uzp1	v0.8b, v0.8b, v0.8b
+uzp1	v0.8h, v0.8h, v0.8h
+uzp2	v0.16b, v0.16b, v0.16b
+uzp2	v0.2d, v0.2d, v0.2d
+uzp2	v0.2s, v0.2s, v0.2s
+uzp2	v0.4h, v0.4h, v0.4h
+uzp2	v0.4s, v0.4s, v0.4s
+uzp2	v0.8b, v0.8b, v0.8b
+uzp2	v0.8h, v0.8h, v0.8h
+xtn v0.2s, v0.2d
+xtn v0.4h, v0.4s
+xtn v0.8b, v0.8h
+xtn2 v0.16b, v0.8h
+xtn2 v0.4s, v0.2d
+xtn2 v0.8h, v0.4s
+zip1	v0.16b, v0.16b, v0.16b
+zip1	v0.2d, v0.2d, v0.2d
+zip1	v0.2s, v0.2s, v0.2s
+zip1	v0.4h, v0.4h, v0.4h
+zip1	v0.4s, v0.4s, v0.4s
+zip1	v0.8b, v0.8b, v0.8b
+zip1	v0.8h, v0.8h, v0.8h
+zip2	v0.16b, v0.16b, v0.16b
+zip2	v0.2d, v0.2d, v0.2d
+zip2	v0.2s, v0.2s, v0.2s
+zip2	v0.4h, v0.4h, v0.4h
+zip2	v0.4s, v0.4s, v0.4s
+zip2	v0.8b, v0.8b, v0.8b
+zip2	v0.8h, v0.8h, v0.8h
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00                        abs	d29, d24
+# CHECK-NEXT:  1      3     1.00                        abs	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        abs	v0.2d, v0.2d
+# CHECK-NEXT:  1      3     1.00                        abs	v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        abs	v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        abs	v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        abs	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        abs	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        add	d17, d31, d29
+# CHECK-NEXT:  1      3     1.00                        add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     1.00                        addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  1      3     1.00                        bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        cls	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        cls	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        cls	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        cls	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        cls	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        cls	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        clz	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        clz	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        clz	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        clz	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        clz	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        clz	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        cmeq	d20, d21, #0
+# CHECK-NEXT:  1      3     1.00                        cmeq	d20, d21, d22
+# CHECK-NEXT:  1      3     1.00                        cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  1      3     1.00                        cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        cmge	d20, d21, #0
+# CHECK-NEXT:  1      3     1.00                        cmge	d20, d21, d22
+# CHECK-NEXT:  1      3     1.00                        cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  1      3     1.00                        cmgt	d20, d21, #0
+# CHECK-NEXT:  1      3     1.00                        cmgt	d20, d21, d22
+# CHECK-NEXT:  1      3     1.00                        cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  1      3     1.00                        cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        cmhi	d20, d21, d22
+# CHECK-NEXT:  1      3     1.00                        cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        cmhs	d20, d21, d22
+# CHECK-NEXT:  1      3     1.00                        cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        cmle	d20, d21, #0
+# CHECK-NEXT:  1      3     1.00                        cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  1      3     1.00                        cmlt	d20, d21, #0
+# CHECK-NEXT:  1      3     1.00                        cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  1      4     1.00                        cmtst	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        cnt	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        cnt	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        dup	v0.16b, w28
+# CHECK-NEXT:  1      3     1.00                        dup	v0.2d, x28
+# CHECK-NEXT:  1      4     1.00                        dup	v0.2s, w28
+# CHECK-NEXT:  1      4     1.00                        dup	v0.4h, w28
+# CHECK-NEXT:  1      3     1.00                        dup	v0.4s, w28
+# CHECK-NEXT:  1      4     1.00                        dup	v0.8b, w28
+# CHECK-NEXT:  1      3     1.00                        dup	v0.8h, w28
+# CHECK-NEXT:  1      3     1.00                        eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        fabd	d29, d24, d20
+# CHECK-NEXT:  1      4     1.00                        fabd	s29, s24, s20
+# CHECK-NEXT:  1      4     1.00                        fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fabs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fabs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fabs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fabs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fabs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        facge	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        facge	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        facgt	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        facgt	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcmge	d20, d21, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        fcmge	s10, s11, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	d20, d21, d22
+# CHECK-NEXT:  1      4     1.00                        fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	s10, s11, s12
+# CHECK-NEXT:  1      4     1.00                        fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcmle	d20, d21, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmle	s10, s11, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fcvtas	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtas	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtau	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtau	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtms	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtms	s22, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtns	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtns	s22, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtps	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtps	s22, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtxn	s22, d13
+# CHECK-NEXT:  1      4     1.00                        fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	d21, d12, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	s21, s12, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	d21, d12, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	d21, d14
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	s12, s13
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	s21, s12, #1
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  1      13    10.00                       fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  1      4     1.00                        fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  1      4     1.00                        fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  1      4     1.00                        fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmulx	d23, d11, d1
+# CHECK-NEXT:  1      4     1.00                        fmulx	s20, s22, s15
+# CHECK-NEXT:  1      4     1.00                        fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fneg	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        fneg	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        fneg	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        fneg	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        fneg	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frecpe	d13, d13
+# CHECK-NEXT:  1      4     1.00                        frecpe	s19, s14
+# CHECK-NEXT:  1      4     1.00                        frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frecps	d22, d30, d21
+# CHECK-NEXT:  1      4     1.00                        frecps	s21, s16, s13
+# CHECK-NEXT:  1      4     1.00                        frecpx	d16, d19
+# CHECK-NEXT:  1      4     1.00                        frecpx	s18, s10
+# CHECK-NEXT:  1      4     1.00                        frinta	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frinta	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frinta	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frinta	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frinta	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frinti	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frinti	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frinti	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frinti	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frinti	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frintm	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frintm	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frintm	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frintm	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frintm	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frintn	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frintn	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frintn	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frintn	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frintn	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frintp	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frintp	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frintp	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frintp	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frintp	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frintx	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frintx	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frintx	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frintx	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frintx	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        frintz	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        frintz	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        frintz	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        frintz	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        frintz	v0.8h, v0.8h
+# CHECK-NEXT:  1      22    19.00                       frsqrte	d21, d12
+# CHECK-NEXT:  1      12    9.00                        frsqrte	s22, s13
+# CHECK-NEXT:  1      22    19.00                       frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  1      12    9.00                        frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  1      8     5.00                        frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  1      12    9.00                        frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  1      8     5.00                        frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  1      22    19.00                       frsqrts	d8, d22, d18
+# CHECK-NEXT:  1      12    9.00                        frsqrts	s21, s5, s12
+# CHECK-NEXT:  1      22    19.00                       frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      22    19.00                       fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  1      12    9.00                        fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  1      8     5.00                        fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  1      12    9.00                        fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  1      8     5.00                        fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00    *                   ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      5     3.00    *                   ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  1      6     4.00    *                   ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      4     2.00    *                   ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  1      5     3.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  2      4     2.00    *                   ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      3     1.00    *                   ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  1      4     2.00    *                   ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  1      3     1.00    *                   ld1	{ v0.b }[9], [x0]
+# CHECK-NEXT:  2      3     1.00    *                   ld1	{ v0.b }[9], [x0], #1
+# CHECK-NEXT:  1      3     1.00    *                   ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      3     1.00    *                   ld1r	{ v0.16b }, [x0], #1
+# CHECK-NEXT:  1      3     1.00    *                   ld1r	{ v0.8h }, [x15]
+# CHECK-NEXT:  2      3     1.00    *                   ld1r	{ v0.8h }, [x15], #2
+# CHECK-NEXT:  2      4     2.00    *                   ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  1      4     1.00    *                   ld2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  1      4     6.00    *                   ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  2      4     6.00    *                   ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  1      3     2.00    *                   ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  2      3     2.00    *                   ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  1      3     2.00    *                   ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  2      3     2.00    *                   ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  1      5     6.00    *                   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      5     6.00    *                   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  1      5     7.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  2      5     7.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  1      4     3.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      4     3.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  1      4     3.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  2      4     3.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  1      5     7.00    *                   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      5     8.00    *                   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  1      6     7.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  2      6     7.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  2      6     7.00    *                   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  1      4     4.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  2      4     4.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  1      4     4.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      4     4.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  1      4     1.00                        mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        mov	b0, v0.b[15]
+# CHECK-NEXT:  1      4     1.00                        mov	d6, v0.d[1]
+# CHECK-NEXT:  1      4     1.00                        mov	h2, v0.h[5]
+# CHECK-NEXT:  1      4     1.00                        mov	s17, v0.s[2]
+# CHECK-NEXT:  1      3     1.00                        mov	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        mov	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  1      4     1.00                        movi	v0.16b, #31
+# CHECK-NEXT:  1      4     1.00                        movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  1      4     1.00                        movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  1      4     1.00                        movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  1      4     1.00                        movi	v0.8b, #255
+# CHECK-NEXT:  1      4     1.00                        mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        mvni	v0.2s, #0
+# CHECK-NEXT:  1      3     1.00                        mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  1      3     1.00                        neg	d29, d24
+# CHECK-NEXT:  1      3     1.00                        neg	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        neg	v0.2d, v0.2d
+# CHECK-NEXT:  1      3     1.00                        neg	v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        neg	v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        neg	v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        neg	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        neg	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        mvn	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        mvn	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        mov	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        orr	v0.8h, #31
+# CHECK-NEXT:  1      4     1.00                        pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      8     1.00                        raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      8     1.00                        raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      8     1.00                        raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     1.00                        raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     1.00                        raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      8     1.00                        raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        rbit	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        rbit	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        rev16	v21.8b, v1.8b
+# CHECK-NEXT:  1      4     1.00                        rev16	v30.16b, v31.16b
+# CHECK-NEXT:  1      4     1.00                        rev32	v0.4h, v9.4h
+# CHECK-NEXT:  1      4     1.00                        rev32	v21.8b, v1.8b
+# CHECK-NEXT:  1      4     1.00                        rev32	v30.16b, v31.16b
+# CHECK-NEXT:  1      4     1.00                        rev32	v4.8h, v7.8h
+# CHECK-NEXT:  1      4     1.00                        rev64	v0.16b, v31.16b
+# CHECK-NEXT:  1      4     1.00                        rev64	v1.8b, v9.8b
+# CHECK-NEXT:  1      4     1.00                        rev64	v13.4h, v21.4h
+# CHECK-NEXT:  1      4     1.00                        rev64	v2.8h, v4.8h
+# CHECK-NEXT:  1      4     1.00                        rev64	v4.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        rev64	v6.4s, v8.4s
+# CHECK-NEXT:  1      4     1.00                        rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      8     1.00                        rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      8     1.00                        rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      8     1.00                        rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     1.00                        rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     1.00                        rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      8     1.00                        rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      6     1.00                        saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      6     1.00                        sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      6     1.00                        sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      6     1.00                        sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      6     1.00                        sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      6     1.00                        sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      6     1.00                        sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  1      7     2.00                        sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  1      3     1.00                        saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  1      3     1.00                        saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     1.00                        scvtf	d21, d12
+# CHECK-NEXT:  1      4     1.00                        scvtf	d21, d12, #64
+# CHECK-NEXT:  1      4     1.00                        scvtf	s22, s13
+# CHECK-NEXT:  1      4     1.00                        scvtf	s22, s13, #32
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        shl	d7, d10, #12
+# CHECK-NEXT:  1      3     1.00                        shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  1      3     1.00                        shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  1      3     1.00                        shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  1      3     1.00                        shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  1      3     1.00                        shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  1      3     1.00                        shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  1      3     1.00                        shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  1      3     1.00                        shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        sli	d10, d14, #12
+# CHECK-NEXT:  1      4     1.00                        sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        sqabs	b19, b14
+# CHECK-NEXT:  1      4     1.00                        sqabs	d18, d12
+# CHECK-NEXT:  1      4     1.00                        sqabs	h21, h15
+# CHECK-NEXT:  1      4     1.00                        sqabs	s20, s12
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqadd	b20, b11, b15
+# CHECK-NEXT:  1      4     1.00                        sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	d19, s24, s12
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	s17, h27, h12
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmull	d15, s22, s12
+# CHECK-NEXT:  1      4     1.00                        sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmull	s12, h22, h12
+# CHECK-NEXT:  1      4     1.00                        sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqneg	b19, b14
+# CHECK-NEXT:  1      4     1.00                        sqneg	d18, d12
+# CHECK-NEXT:  1      4     1.00                        sqneg	h21, h15
+# CHECK-NEXT:  1      4     1.00                        sqneg	s20, s12
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqrshl	d31, d31, d31
+# CHECK-NEXT:  1      4     1.00                        sqrshl	h3, h4, h15
+# CHECK-NEXT:  1      4     1.00                        sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	b10, h13, #2
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	h15, s10, #6
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	s15, d12, #9
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	b17, h10, #6
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	h10, s13, #15
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	s22, d16, #31
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	b11, b19, #7
+# CHECK-NEXT:  1      4     1.00                        sqshl	d15, d16, #51
+# CHECK-NEXT:  1      4     1.00                        sqshl	d31, d31, d31
+# CHECK-NEXT:  1      4     1.00                        sqshl	h13, h18, #11
+# CHECK-NEXT:  1      4     1.00                        sqshl	h3, h4, h15
+# CHECK-NEXT:  1      4     1.00                        sqshl	s14, s17, #22
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	b15, b18, #6
+# CHECK-NEXT:  1      4     1.00                        sqshlu	d11, d13, #32
+# CHECK-NEXT:  1      4     1.00                        sqshlu	h19, h17, #6
+# CHECK-NEXT:  1      4     1.00                        sqshlu	s16, s14, #25
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn	b10, h15, #5
+# CHECK-NEXT:  1      4     1.00                        sqshrn	h17, s10, #4
+# CHECK-NEXT:  1      4     1.00                        sqshrn	s18, d10, #31
+# CHECK-NEXT:  1      4     1.00                        sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun	b15, h10, #7
+# CHECK-NEXT:  1      4     1.00                        sqshrun	h20, s14, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun	s10, d15, #15
+# CHECK-NEXT:  1      4     1.00                        sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sqsub	s20, s10, s7
+# CHECK-NEXT:  1      4     1.00                        sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        sqxtn	b18, h18
+# CHECK-NEXT:  1      4     1.00                        sqxtn	h20, s17
+# CHECK-NEXT:  1      4     1.00                        sqxtn	s19, d14
+# CHECK-NEXT:  1      4     1.00                        sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqxtun	b19, h14
+# CHECK-NEXT:  1      4     1.00                        sqxtun	h21, s15
+# CHECK-NEXT:  1      4     1.00                        sqxtun	s20, d12
+# CHECK-NEXT:  1      4     1.00                        sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  1      3     1.00                        srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        sri	d10, d12, #14
+# CHECK-NEXT:  1      4     1.00                        sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        srshl	d16, d16, d16
+# CHECK-NEXT:  1      3     1.00                        srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        srshr	d19, d18, #7
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	d15, d11, #19
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      7     2.00                        srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        sshl	d31, d31, d31
+# CHECK-NEXT:  1      3     1.00                        sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     1.00                        sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	d15, d16, #12
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	d18, d12, #21
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      5     2.00           *            st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      5     2.00           *            st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  1      5     4.00           *            st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      5     2.00           *            st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  1      5     2.00           *            st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  2      5     4.00           *            st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      5     2.00           *            st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  1      5     2.00           *            st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  1      4     1.00           *            st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  2      4     1.00           *            st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  2      5     4.00           *            st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  1      5     2.00           *            st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  1      5     2.00           *            st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  2      5     2.00           *            st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  1      5     4.00           *            st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      5     4.00           *            st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  1      5     2.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  2      5     2.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  1      5     4.00           *            st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      5     4.00           *            st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  1      5     2.00           *            st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
+# CHECK-NEXT:  2      5     2.00           *            st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+# CHECK-NEXT:  1      3     1.00                        sub	d15, d5, d16
+# CHECK-NEXT:  1      3     1.00                        sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        suqadd	b19, b14
+# CHECK-NEXT:  1      4     1.00                        suqadd	d18, d22
+# CHECK-NEXT:  1      4     1.00                        suqadd	h20, h15
+# CHECK-NEXT:  1      4     1.00                        suqadd	s21, s12
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      6     1.00                        uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      6     1.00                        uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      6     1.00                        uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      6     1.00                        uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      6     1.00                        uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      6     1.00                        uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      6     1.00                        uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  1      7     2.00                        uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  1      3     1.00                        uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  1      3     1.00                        uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d21, d14
+# CHECK-NEXT:  1      4     1.00                        ucvtf	d21, d14, #64
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s22, s13
+# CHECK-NEXT:  1      4     1.00                        ucvtf	s22, s13, #32
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uqadd	h0, h1, h5
+# CHECK-NEXT:  1      4     1.00                        uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uqrshl	b11, b20, b30
+# CHECK-NEXT:  1      4     1.00                        uqrshl	s23, s20, s16
+# CHECK-NEXT:  1      4     1.00                        uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	b10, h12, #5
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	h12, s10, #14
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	s10, d10, #25
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	b11, b20, b30
+# CHECK-NEXT:  1      4     1.00                        uqshl	b18, b15, #6
+# CHECK-NEXT:  1      4     1.00                        uqshl	d15, d12, #19
+# CHECK-NEXT:  1      4     1.00                        uqshl	h11, h18, #7
+# CHECK-NEXT:  1      4     1.00                        uqshl	s14, s19, #18
+# CHECK-NEXT:  1      4     1.00                        uqshl	s23, s20, s16
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uqshrn	b12, h10, #7
+# CHECK-NEXT:  1      4     1.00                        uqshrn	h10, s14, #5
+# CHECK-NEXT:  1      4     1.00                        uqshrn	s10, d12, #13
+# CHECK-NEXT:  1      4     1.00                        uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     1.00                        uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     1.00                        uqsub	d16, d16, d16
+# CHECK-NEXT:  1      4     1.00                        uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        uqxtn	b18, h18
+# CHECK-NEXT:  1      4     1.00                        uqxtn	h20, s17
+# CHECK-NEXT:  1      4     1.00                        uqxtn	s19, d14
+# CHECK-NEXT:  1      4     1.00                        uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        urshl	d8, d7, d4
+# CHECK-NEXT:  1      3     1.00                        urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     1.00                        urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        urshr	d20, d23, #31
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      12    9.00                        ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  1      12    9.00                        ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  1      7     2.00                        ursra	d18, d10, #13
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      7     2.00                        ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        ushl	d0, d0, d0
+# CHECK-NEXT:  1      3     1.00                        ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	d10, d17, #18
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     1.00                        usqadd	b19, b14
+# CHECK-NEXT:  1      4     1.00                        usqadd	d18, d22
+# CHECK-NEXT:  1      4     1.00                        usqadd	h20, h15
+# CHECK-NEXT:  1      4     1.00                        usqadd	s21, s12
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        usra	d20, d13, #61
+# CHECK-NEXT:  1      3     1.00                        usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     1.00                        usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     1.00                        usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     1.00                        usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     1.00                        usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     1.00                        usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     1.00                        usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     1.00                        usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     1.00                        usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      3     1.00                        usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      3     1.00                        usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      3     1.00                        usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      3     1.00                        usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      3     1.00                        usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        xtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        xtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        xtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     1.00                        xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     1.00                        xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     1.00                        zip2	v0.8h, v0.8h, v0.8h
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA320UnitALU
+# CHECK-NEXT: [1]   - CortexA320UnitB
+# CHECK-NEXT: [2]   - CortexA320UnitDiv
+# CHECK-NEXT: [3]   - CortexA320UnitLdSt
+# CHECK-NEXT: [4]   - CortexA320UnitMAC
+# CHECK-NEXT: [5]   - CortexA320UnitPAC
+# CHECK-NEXT: [6]   - CortexA320UnitVALU
+# CHECK-NEXT: [7]   - CortexA320UnitVMAC
+# CHECK-NEXT: [8]   - CortexA320UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT:  -      -      -     193.00  -      -     1002.00 6.00  197.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	d17, d31, d29
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmeq	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmge	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmgt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmhi	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmhs	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmle	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmlt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmtst	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.16b, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.2d, x28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.2s, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.4h, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.4s, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.8b, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dup	v0.8h, w28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	d29, d24, d20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	s29, s24, s20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtxn	s22, d13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     10.00  fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	d23, d11, d1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	s20, s22, s15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	d13, d13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	s19, s14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecps	d22, d30, d21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecps	s21, s16, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpx	d16, d19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frecpx	s18, s10
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  frsqrte	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   frsqrte	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  frsqrts	d8, d22, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   frsqrts	s21, s5, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1	{ v0.b }[9], [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1	{ v0.b }[9], [x0], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1r	{ v0.16b }, [x0], #1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1r	{ v0.8h }, [x15]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1r	{ v0.8h }, [x15], #2
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -     6.00    -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  -      -      -     6.00    -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  -      -      -     6.00    -      -      -      -      -     ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -     6.00    -      -      -      -      -     ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	b0, v0.b[15]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	d6, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	h2, v0.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	s17, v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	v0.16b, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movi	v0.8b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mvni	v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mvn	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mvn	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	v0.8h, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev16	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev16	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev32	v0.4h, v9.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev32	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev32	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev32	v4.8h, v7.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v0.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v1.8b, v9.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v13.4h, v21.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v2.8h, v4.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v4.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev64	v6.4s, v8.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	d21, d12, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shl	d7, d10, #12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	d10, d14, #12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	b20, b11, b15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	d19, s24, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	s17, h27, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	d15, s22, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	s12, h22, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	b10, h13, #2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	h15, s10, #6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	s15, d12, #9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	b17, h10, #6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	h10, s13, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	s22, d16, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	b11, b19, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	d15, d16, #51
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	h13, h18, #11
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	s14, s17, #22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	b15, b18, #6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	d11, d13, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	h19, h17, #6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	s16, s14, #25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	b10, h15, #5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	h17, s10, #4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	s18, d10, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	b15, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	h20, s14, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	s10, d15, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	s20, s10, s7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	b19, h14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	h21, s15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	s20, d12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	d10, d12, #14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	d19, d18, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	d15, d11, #19
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	d15, d16, #12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	d18, d12, #21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -     4.00    -      -      -      -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	d15, d5, d16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	d21, d14, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	h0, h1, h5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	b10, h12, #5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	h12, s10, #14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	s10, d10, #25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	b18, b15, #6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	d15, d12, #19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	h11, h18, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	s14, s19, #18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	b12, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	h10, s14, #5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	s10, d12, #13
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	d8, d7, d4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	d20, d23, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	d18, d10, #13
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushl	d0, d0, d0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	d10, d17, #18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	d20, d13, #61
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	v0.8h, v0.8h, v0.8h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-sve-instructions.s
new file mode 100644
index 000000000000..ad8d57bdb32d
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A320-sve-instructions.s
@@ -0,0 +1,10258 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a320 -mattr=+sve2-aes,+sve2-sha3,+sve2-sm4 -instruction-tables < %s | FileCheck %s
+
+abs	z0.b, p0/m, z0.b
+abs	z0.d, p0/m, z0.d
+abs	z0.h, p0/m, z0.h
+abs	z0.s, p0/m, z0.s
+abs	z31.b, p7/m, z31.b
+abs	z31.d, p7/m, z31.d
+abs	z31.h, p7/m, z31.h
+abs	z31.s, p7/m, z31.s
+adclb	z0.d, z1.d, z31.d
+adclb	z0.s, z1.s, z31.s
+adclt	z0.d, z1.d, z31.d
+adclt	z0.s, z1.s, z31.s
+add	z0.b, p0/m, z0.b, z0.b
+add	z0.b, z0.b, #0
+add	z0.b, z0.b, z0.b
+add	z0.d, p0/m, z0.d, z0.d
+add	z0.d, z0.d, #0
+add	z0.d, z0.d, #0, lsl #8
+add	z0.d, z0.d, z0.d
+add	z0.h, p0/m, z0.h, z0.h
+add	z0.h, z0.h, #0
+add	z0.h, z0.h, #0, lsl #8
+add	z0.h, z0.h, z0.h
+add	z0.s, p0/m, z0.s, z0.s
+add	z0.s, z0.s, #0
+add	z0.s, z0.s, #0, lsl #8
+add	z0.s, z0.s, z0.s
+add	z0.s, z1.s, z2.s
+add	z21.b, p5/m, z21.b, z10.b
+add	z21.b, z10.b, z21.b
+add	z21.d, p5/m, z21.d, z10.d
+add	z21.d, z10.d, z21.d
+add	z21.h, p5/m, z21.h, z10.h
+add	z21.h, z10.h, z21.h
+add	z21.s, p5/m, z21.s, z10.s
+add	z21.s, z10.s, z21.s
+add	z23.b, p3/m, z23.b, z13.b
+add	z23.b, z13.b, z8.b
+add	z23.d, p3/m, z23.d, z13.d
+add	z23.d, z13.d, z8.d
+add	z23.h, p3/m, z23.h, z13.h
+add	z23.h, z13.h, z8.h
+add	z23.s, p3/m, z23.s, z13.s
+add	z23.s, z13.s, z8.s
+add	z31.b, p7/m, z31.b, z31.b
+add	z31.b, z31.b, #255
+add	z31.b, z31.b, z31.b
+add	z31.d, p7/m, z31.d, z31.d
+add	z31.d, z31.d, #65280
+add	z31.d, z31.d, z31.d
+add	z31.h, p7/m, z31.h, z31.h
+add	z31.h, z31.h, #65280
+add	z31.h, z31.h, z31.h
+add	z31.s, p7/m, z31.s, z31.s
+add	z31.s, z31.s, #65280
+add	z31.s, z31.s, z31.s
+addhnb	z0.b, z1.h, z31.h
+addhnb	z0.h, z1.s, z31.s
+addhnb	z0.s, z1.d, z31.d
+addhnt	z0.b, z1.h, z31.h
+addhnt	z0.h, z1.s, z31.s
+addhnt	z0.s, z1.d, z31.d
+addp	z0.b, p0/m, z0.b, z1.b
+addp	z0.h, p0/m, z0.h, z1.h
+addp	z29.s, p7/m, z29.s, z30.s
+addp	z31.d, p7/m, z31.d, z30.d
+addpl	sp, sp, #31
+addpl	x0, x0, #-32
+addpl	x21, x21, #0
+addpl	x23, x8, #-1
+addvl	sp, sp, #31
+addvl	x0, x0, #-32
+addvl	x21, x21, #0
+addvl	x23, x8, #-1
+adr	z0.d, [z0.d, z0.d, lsl #1]
+adr	z0.d, [z0.d, z0.d, lsl #2]
+adr	z0.d, [z0.d, z0.d, lsl #3]
+adr	z0.d, [z0.d, z0.d, sxtw #1]
+adr	z0.d, [z0.d, z0.d, sxtw #2]
+adr	z0.d, [z0.d, z0.d, sxtw #3]
+adr	z0.d, [z0.d, z0.d, sxtw]
+adr	z0.d, [z0.d, z0.d, uxtw #1]
+adr	z0.d, [z0.d, z0.d, uxtw #2]
+adr	z0.d, [z0.d, z0.d, uxtw #3]
+adr	z0.d, [z0.d, z0.d, uxtw]
+adr	z0.d, [z0.d, z0.d]
+adr	z0.s, [z0.s, z0.s, lsl #1]
+adr	z0.s, [z0.s, z0.s, lsl #2]
+adr	z0.s, [z0.s, z0.s, lsl #3]
+adr	z0.s, [z0.s, z0.s]
+aesd	z0.b, z0.b, z31.b
+aese	z0.b, z0.b, z31.b
+aesimc	z0.b, z0.b
+aesimc	z31.b, z31.b
+aesmc	z0.b, z0.b
+aesmc	z31.b, z31.b
+and	p0.b, p0/z, p0.b, p1.b
+and	z0.d, z0.d, #0x6
+and	z0.d, z0.d, #0xfffffffffffffff9
+and	z0.d, z0.d, z0.d
+and	z0.s, z0.s, #0x6
+and	z0.s, z0.s, #0xfffffff9
+and	z23.d, z13.d, z8.d
+and	z23.h, z23.h, #0x6
+and	z23.h, z23.h, #0xfff9
+and	z31.b, p7/m, z31.b, z31.b
+and	z31.d, p7/m, z31.d, z31.d
+and	z31.h, p7/m, z31.h, z31.h
+and	z31.s, p7/m, z31.s, z31.s
+and	z5.b, z5.b, #0x6
+and	z5.b, z5.b, #0xf9
+ands	p0.b, p0/z, p0.b, p1.b
+andv	b0, p7, z31.b
+andv	d0, p7, z31.d
+andv	h0, p7, z31.h
+andv	s0, p7, z31.s
+asr	z0.b, p0/m, z0.b, #1
+asr	z0.b, p0/m, z0.b, z0.b
+asr	z0.b, p0/m, z0.b, z1.d
+asr	z0.b, z0.b, #1
+asr	z0.b, z1.b, z2.d
+asr	z0.d, p0/m, z0.d, #1
+asr	z0.d, p0/m, z0.d, z0.d
+asr	z0.d, z0.d, #1
+asr	z0.h, p0/m, z0.h, #1
+asr	z0.h, p0/m, z0.h, z0.h
+asr	z0.h, p0/m, z0.h, z1.d
+asr	z0.h, z0.h, #1
+asr	z0.h, z1.h, z2.d
+asr	z0.s, p0/m, z0.s, #1
+asr	z0.s, p0/m, z0.s, z0.s
+asr	z0.s, p0/m, z0.s, z1.d
+asr	z0.s, z0.s, #1
+asr	z0.s, z1.s, z2.d
+asr	z31.b, p0/m, z31.b, #8
+asr	z31.b, z31.b, #8
+asr	z31.d, p0/m, z31.d, #64
+asr	z31.d, z31.d, #64
+asr	z31.h, p0/m, z31.h, #16
+asr	z31.h, z31.h, #16
+asr	z31.s, p0/m, z31.s, #32
+asr	z31.s, z31.s, #32
+asrd	z0.b, p0/m, z0.b, #1
+asrd	z0.d, p0/m, z0.d, #1
+asrd	z0.h, p0/m, z0.h, #1
+asrd	z0.s, p0/m, z0.s, #1
+asrd	z31.b, p0/m, z31.b, #8
+asrd	z31.d, p0/m, z31.d, #64
+asrd	z31.h, p0/m, z31.h, #16
+asrd	z31.s, p0/m, z31.s, #32
+asrr	z0.b, p0/m, z0.b, z0.b
+asrr	z0.d, p0/m, z0.d, z0.d
+asrr	z0.h, p0/m, z0.h, z0.h
+asrr	z0.s, p0/m, z0.s, z0.s
+bcax	z29.d, z29.d, z30.d, z31.d
+bdep	z0.b, z1.b, z31.b
+bdep	z0.d, z1.d, z31.d
+bdep	z0.h, z1.h, z31.h
+bdep	z0.s, z1.s, z31.s
+bext	z0.b, z1.b, z31.b
+bext	z0.d, z1.d, z31.d
+bext	z0.h, z1.h, z31.h
+bext	z0.s, z1.s, z31.s
+bfcvt	z0.h, p0/m, z1.s
+bfcvtnt	z0.h, p0/m, z1.s
+bfdot	z0.s, z1.h, z2.h
+bfdot	z0.s, z1.h, z2.h[0]
+bfdot	z0.s, z1.h, z2.h[3]
+bfmlalb	z0.s, z1.h, z2.h
+bfmlalb	z0.s, z1.h, z2.h[0]
+bfmlalb	z0.s, z1.h, z2.h[7]
+bfmlalb	z10.s, z21.h, z14.h
+bfmlalb	z21.s, z14.h, z3.h[2]
+bfmlalt	z0.s, z1.h, z2.h
+bfmlalt	z0.s, z1.h, z2.h[0]
+bfmlalt	z0.s, z1.h, z2.h[7]
+bfmlalt	z0.s, z1.h, z7.h[7]
+bfmlalt	z14.s, z10.h, z21.h
+bfmmla	z0.s, z1.h, z2.h
+bgrp	z0.b, z1.b, z31.b
+bgrp	z0.d, z1.d, z31.d
+bgrp	z0.h, z1.h, z31.h
+bgrp	z0.s, z1.s, z31.s
+bic	p0.b, p0/z, p0.b, p0.b
+bic	p15.b, p15/z, p15.b, p15.b
+bic	z0.d, z0.d, z0.d
+bic	z23.d, z13.d, z8.d
+bic	z31.b, p7/m, z31.b, z31.b
+bic	z31.d, p7/m, z31.d, z31.d
+bic	z31.h, p7/m, z31.h, z31.h
+bic	z31.s, p7/m, z31.s, z31.s
+bics	p0.b, p0/z, p0.b, p0.b
+bics	p15.b, p15/z, p15.b, p15.b
+brka	p0.b, p15/m, p15.b
+brka	p0.b, p15/z, p15.b
+brkas	p0.b, p15/z, p15.b
+brkb	p0.b, p15/m, p15.b
+brkb	p0.b, p15/z, p15.b
+brkbs	p0.b, p15/z, p15.b
+brkn	p0.b, p15/z, p1.b, p0.b
+brkn	p15.b, p15/z, p15.b, p15.b
+brkns	p0.b, p15/z, p1.b, p0.b
+brkns	p15.b, p15/z, p15.b, p15.b
+brkpa	p0.b, p15/z, p1.b, p2.b
+brkpa	p15.b, p15/z, p15.b, p15.b
+brkpas	p0.b, p15/z, p1.b, p2.b
+brkpas	p15.b, p15/z, p15.b, p15.b
+brkpb	p0.b, p15/z, p1.b, p2.b
+brkpb	p15.b, p15/z, p15.b, p15.b
+brkpbs	p0.b, p15/z, p1.b, p2.b
+brkpbs	p15.b, p15/z, p15.b, p15.b
+bsl	z0.d, z0.d, z1.d, z2.d
+bsl1n	z0.d, z0.d, z1.d, z2.d
+bsl2n	z0.d, z0.d, z1.d, z2.d
+cadd	z0.b, z0.b, z0.b, #90
+cadd	z0.d, z0.d, z0.d, #90
+cadd	z0.h, z0.h, z0.h, #90
+cadd	z0.s, z0.s, z0.s, #90
+cadd	z31.b, z31.b, z31.b, #270
+cadd	z31.d, z31.d, z31.d, #270
+cadd	z31.h, z31.h, z31.h, #270
+cadd	z31.s, z31.s, z31.s, #270
+cdot	z0.d, z1.h, z15.h[1], #0
+cdot	z0.d, z1.h, z31.h, #0
+cdot	z0.d, z1.h, z31.h, #180
+cdot	z0.d, z1.h, z31.h, #270
+cdot	z0.d, z1.h, z31.h, #90
+cdot	z0.s, z1.b, z31.b, #0
+cdot	z0.s, z1.b, z7.b[3], #0
+cdot	z29.d, z30.h, z0.h[0], #180
+cdot	z31.d, z30.h, z7.h[1], #270
+cdot	z5.d, z6.h, z3.h[0], #90
+clasta	b0, p7, b0, z31.b
+clasta	d0, p7, d0, z31.d
+clasta	h0, p7, h0, z31.h
+clasta	s0, p7, s0, z31.s
+clasta	w0, p7, w0, z31.b
+clasta	w0, p7, w0, z31.h
+clasta	w0, p7, w0, z31.s
+clasta	x0, p7, x0, z31.d
+clasta	z0.b, p7, z0.b, z31.b
+clasta	z0.d, p7, z0.d, z31.d
+clasta	z0.h, p7, z0.h, z31.h
+clasta	z0.s, p7, z0.s, z31.s
+clastb	b0, p7, b0, z31.b
+clastb	d0, p7, d0, z31.d
+clastb	h0, p7, h0, z31.h
+clastb	s0, p7, s0, z31.s
+clastb	w0, p7, w0, z31.b
+clastb	w0, p7, w0, z31.h
+clastb	w0, p7, w0, z31.s
+clastb	x0, p7, x0, z31.d
+clastb	z0.b, p7, z0.b, z31.b
+clastb	z0.d, p7, z0.d, z31.d
+clastb	z0.h, p7, z0.h, z31.h
+clastb	z0.s, p7, z0.s, z31.s
+cls	z31.b, p7/m, z31.b
+cls	z31.d, p7/m, z31.d
+cls	z31.h, p7/m, z31.h
+cls	z31.s, p7/m, z31.s
+clz	z31.b, p7/m, z31.b
+clz	z31.d, p7/m, z31.d
+clz	z31.h, p7/m, z31.h
+clz	z31.s, p7/m, z31.s
+cmla	z0.b, z1.b, z2.b, #0
+cmla	z0.d, z1.d, z2.d, #0
+cmla	z0.h, z1.h, z2.h, #0
+cmla	z0.h, z1.h, z2.h[0], #0
+cmla	z0.s, z1.s, z2.s, #0
+cmla	z0.s, z1.s, z2.s[0], #0
+cmla	z15.b, z16.b, z17.b, #270
+cmla	z15.d, z16.d, z17.d, #270
+cmla	z15.h, z16.h, z17.h, #270
+cmla	z15.s, z16.s, z17.s, #270
+cmla	z29.b, z30.b, z31.b, #90
+cmla	z29.d, z30.d, z31.d, #90
+cmla	z29.h, z30.h, z31.h, #90
+cmla	z29.s, z30.s, z31.s, #90
+cmla	z31.b, z31.b, z31.b, #180
+cmla	z31.d, z31.d, z31.d, #180
+cmla	z31.h, z30.h, z7.h[0], #180
+cmla	z31.h, z31.h, z31.h, #180
+cmla	z31.s, z30.s, z7.s[0], #180
+cmla	z31.s, z31.s, z31.s, #180
+cmpeq	p0.b, p0/z, z0.b, #-16
+cmpeq	p0.b, p0/z, z0.b, #15
+cmpeq	p0.b, p0/z, z0.b, z0.b
+cmpeq	p0.b, p0/z, z0.b, z0.d
+cmpeq	p0.d, p0/z, z0.d, #-16
+cmpeq	p0.d, p0/z, z0.d, #15
+cmpeq	p0.d, p0/z, z0.d, z0.d
+cmpeq	p0.h, p0/z, z0.h, #-16
+cmpeq	p0.h, p0/z, z0.h, #15
+cmpeq	p0.h, p0/z, z0.h, z0.d
+cmpeq	p0.h, p0/z, z0.h, z0.h
+cmpeq	p0.s, p0/z, z0.s, #-16
+cmpeq	p0.s, p0/z, z0.s, #15
+cmpeq	p0.s, p0/z, z0.s, z0.d
+cmpeq	p0.s, p0/z, z0.s, z0.s
+cmpge	p0.b, p0/z, z0.b, #-16
+cmpge	p0.b, p0/z, z0.b, #15
+cmpge	p0.b, p0/z, z0.b, z0.b
+cmpge	p0.b, p0/z, z0.b, z0.d
+cmpge	p0.b, p0/z, z1.b, z0.b
+cmpge	p0.d, p0/z, z0.d, #-16
+cmpge	p0.d, p0/z, z0.d, #15
+cmpge	p0.d, p0/z, z0.d, z0.d
+cmpge	p0.d, p0/z, z1.d, z0.d
+cmpge	p0.h, p0/z, z0.h, #-16
+cmpge	p0.h, p0/z, z0.h, #15
+cmpge	p0.h, p0/z, z0.h, z0.d
+cmpge	p0.h, p0/z, z0.h, z0.h
+cmpge	p0.h, p0/z, z1.h, z0.h
+cmpge	p0.s, p0/z, z0.s, #-16
+cmpge	p0.s, p0/z, z0.s, #15
+cmpge	p0.s, p0/z, z0.s, z0.d
+cmpge	p0.s, p0/z, z0.s, z0.s
+cmpge	p0.s, p0/z, z1.s, z0.s
+cmpgt	p0.b, p0/z, z0.b, #-16
+cmpgt	p0.b, p0/z, z0.b, #15
+cmpgt	p0.b, p0/z, z0.b, z0.b
+cmpgt	p0.b, p0/z, z0.b, z0.d
+cmpgt	p0.b, p0/z, z1.b, z0.b
+cmpgt	p0.d, p0/z, z0.d, #-16
+cmpgt	p0.d, p0/z, z0.d, #15
+cmpgt	p0.d, p0/z, z0.d, z0.d
+cmpgt	p0.d, p0/z, z1.d, z0.d
+cmpgt	p0.h, p0/z, z0.h, #-16
+cmpgt	p0.h, p0/z, z0.h, #15
+cmpgt	p0.h, p0/z, z0.h, z0.d
+cmpgt	p0.h, p0/z, z0.h, z0.h
+cmpgt	p0.h, p0/z, z1.h, z0.h
+cmpgt	p0.s, p0/z, z0.s, #-16
+cmpgt	p0.s, p0/z, z0.s, #15
+cmpgt	p0.s, p0/z, z0.s, z0.d
+cmpgt	p0.s, p0/z, z0.s, z0.s
+cmpgt	p0.s, p0/z, z1.s, z0.s
+cmphi	p0.b, p0/z, z0.b, #0
+cmphi	p0.b, p0/z, z0.b, #127
+cmphi	p0.b, p0/z, z0.b, z0.b
+cmphi	p0.b, p0/z, z0.b, z0.d
+cmphi	p0.b, p0/z, z1.b, z0.b
+cmphi	p0.d, p0/z, z0.d, #0
+cmphi	p0.d, p0/z, z0.d, #127
+cmphi	p0.d, p0/z, z0.d, z0.d
+cmphi	p0.d, p0/z, z1.d, z0.d
+cmphi	p0.h, p0/z, z0.h, #0
+cmphi	p0.h, p0/z, z0.h, #127
+cmphi	p0.h, p0/z, z0.h, z0.d
+cmphi	p0.h, p0/z, z0.h, z0.h
+cmphi	p0.h, p0/z, z1.h, z0.h
+cmphi	p0.s, p0/z, z0.s, #0
+cmphi	p0.s, p0/z, z0.s, #127
+cmphi	p0.s, p0/z, z0.s, z0.d
+cmphi	p0.s, p0/z, z0.s, z0.s
+cmphi	p0.s, p0/z, z1.s, z0.s
+cmphs	p0.b, p0/z, z0.b, #0
+cmphs	p0.b, p0/z, z0.b, #127
+cmphs	p0.b, p0/z, z0.b, z0.b
+cmphs	p0.b, p0/z, z0.b, z0.d
+cmphs	p0.b, p0/z, z1.b, z0.b
+cmphs	p0.d, p0/z, z0.d, #0
+cmphs	p0.d, p0/z, z0.d, #127
+cmphs	p0.d, p0/z, z0.d, z0.d
+cmphs	p0.d, p0/z, z1.d, z0.d
+cmphs	p0.h, p0/z, z0.h, #0
+cmphs	p0.h, p0/z, z0.h, #127
+cmphs	p0.h, p0/z, z0.h, z0.d
+cmphs	p0.h, p0/z, z0.h, z0.h
+cmphs	p0.h, p0/z, z1.h, z0.h
+cmphs	p0.s, p0/z, z0.s, #0
+cmphs	p0.s, p0/z, z0.s, #127
+cmphs	p0.s, p0/z, z0.s, z0.d
+cmphs	p0.s, p0/z, z0.s, z0.s
+cmphs	p0.s, p0/z, z1.s, z0.s
+cmple	p0.b, p0/z, z0.b, #-16
+cmple	p0.b, p0/z, z0.b, #15
+cmple	p0.b, p0/z, z0.b, z0.d
+cmple	p0.d, p0/z, z0.d, #-16
+cmple	p0.d, p0/z, z0.d, #15
+cmple	p0.h, p0/z, z0.h, #-16
+cmple	p0.h, p0/z, z0.h, #15
+cmple	p0.h, p0/z, z0.h, z0.d
+cmple	p0.s, p0/z, z0.s, #-16
+cmple	p0.s, p0/z, z0.s, #15
+cmple	p0.s, p0/z, z0.s, z0.d
+cmplo	p0.b, p0/z, z0.b, #0
+cmplo	p0.b, p0/z, z0.b, #127
+cmplo	p0.b, p0/z, z0.b, z0.d
+cmplo	p0.d, p0/z, z0.d, #0
+cmplo	p0.d, p0/z, z0.d, #127
+cmplo	p0.h, p0/z, z0.h, #0
+cmplo	p0.h, p0/z, z0.h, #127
+cmplo	p0.h, p0/z, z0.h, z0.d
+cmplo	p0.s, p0/z, z0.s, #0
+cmplo	p0.s, p0/z, z0.s, #127
+cmplo	p0.s, p0/z, z0.s, z0.d
+cmpls	p0.b, p0/z, z0.b, #0
+cmpls	p0.b, p0/z, z0.b, #127
+cmpls	p0.b, p0/z, z0.b, z0.d
+cmpls	p0.d, p0/z, z0.d, #0
+cmpls	p0.d, p0/z, z0.d, #127
+cmpls	p0.h, p0/z, z0.h, #0
+cmpls	p0.h, p0/z, z0.h, #127
+cmpls	p0.h, p0/z, z0.h, z0.d
+cmpls	p0.s, p0/z, z0.s, #0
+cmpls	p0.s, p0/z, z0.s, #127
+cmpls	p0.s, p0/z, z0.s, z0.d
+cmplt	p0.b, p0/z, z0.b, #-16
+cmplt	p0.b, p0/z, z0.b, #15
+cmplt	p0.b, p0/z, z0.b, z0.d
+cmplt	p0.d, p0/z, z0.d, #-16
+cmplt	p0.d, p0/z, z0.d, #15
+cmplt	p0.h, p0/z, z0.h, #-16
+cmplt	p0.h, p0/z, z0.h, #15
+cmplt	p0.h, p0/z, z0.h, z0.d
+cmplt	p0.s, p0/z, z0.s, #-16
+cmplt	p0.s, p0/z, z0.s, #15
+cmplt	p0.s, p0/z, z0.s, z0.d
+cmpne	p0.b, p0/z, z0.b, #-16
+cmpne	p0.b, p0/z, z0.b, #15
+cmpne	p0.b, p0/z, z0.b, z0.b
+cmpne	p0.b, p0/z, z0.b, z0.d
+cmpne	p0.d, p0/z, z0.d, #-16
+cmpne	p0.d, p0/z, z0.d, #15
+cmpne	p0.d, p0/z, z0.d, z0.d
+cmpne	p0.h, p0/z, z0.h, #-16
+cmpne	p0.h, p0/z, z0.h, #15
+cmpne	p0.h, p0/z, z0.h, z0.d
+cmpne	p0.h, p0/z, z0.h, z0.h
+cmpne	p0.s, p0/z, z0.s, #-16
+cmpne	p0.s, p0/z, z0.s, #15
+cmpne	p0.s, p0/z, z0.s, z0.d
+cmpne	p0.s, p0/z, z0.s, z0.s
+cnot	z31.b, p7/m, z31.b
+cnot	z31.d, p7/m, z31.d
+cnot	z31.h, p7/m, z31.h
+cnot	z31.s, p7/m, z31.s
+cnt	z31.b, p7/m, z31.b
+cnt	z31.d, p7/m, z31.d
+cnt	z31.h, p7/m, z31.h
+cnt	z31.s, p7/m, z31.s
+cntb	x0
+cntb	x0, #28
+cntb	x0, all, mul #16
+cntb	x0, pow2
+cntd	x0
+cntd	x0, #28
+cntd	x0, all, mul #16
+cntd	x0, pow2
+cnth	x0
+cnth	x0, #28
+cnth	x0, all, mul #16
+cnth	x0, pow2
+cntp	x0, p15, p0.b
+cntp	x0, p15, p0.d
+cntp	x0, p15, p0.h
+cntp	x0, p15, p0.s
+cntw	x0
+cntw	x0, #28
+cntw	x0, all, mul #16
+cntw	x0, pow2
+compact	z31.d, p7, z31.d
+compact	z31.s, p7, z31.s
+ctermeq	w30, wzr
+ctermeq	wzr, w30
+ctermeq	x30, xzr
+ctermeq	xzr, x30
+ctermne	w30, wzr
+ctermne	wzr, w30
+ctermne	x30, xzr
+ctermne	xzr, x30
+decb	x0
+decb	x0, #14
+decb	x0, all, mul #16
+decb	x0, pow2
+decb	x0, vl1
+decd	x0
+decd	x0, #14
+decd	x0, all, mul #16
+decd	x0, pow2
+decd	x0, vl1
+dech	x0
+dech	x0, #14
+dech	x0, all, mul #16
+dech	x0, pow2
+dech	x0, vl1
+decp	x0, p0.b
+decp	x0, p0.d
+decp	x0, p0.h
+decp	x0, p0.s
+decp	xzr, p15.b
+decp	xzr, p15.d
+decp	xzr, p15.h
+decp	xzr, p15.s
+decp	z31.d, p15.d
+decp	z31.h, p15.h
+decp	z31.s, p15.s
+decw	x0
+decw	x0, #14
+decw	x0, all, mul #16
+decw	x0, pow2
+decw	x0, vl1
+dupm	z0.d, #0xfffffffffffffff9
+dupm	z0.s, #0xfffffff9
+dupm	z23.h, #0xfff9
+dupm	z5.b, #0xf9
+eor	p0.b, p0/z, p0.b, p1.b
+eor	z0.d, z0.d, #0x6
+eor	z0.d, z0.d, #0xfffffffffffffff9
+eor	z0.d, z0.d, z0.d
+eor	z0.s, z0.s, #0x6
+eor	z0.s, z0.s, #0xfffffff9
+eor	z23.d, z13.d, z8.d
+eor	z23.h, z23.h, #0x6
+eor	z23.h, z23.h, #0xfff9
+eor	z31.b, p7/m, z31.b, z31.b
+eor	z31.d, p7/m, z31.d, z31.d
+eor	z31.h, p7/m, z31.h, z31.h
+eor	z31.s, p7/m, z31.s, z31.s
+eor	z5.b, z5.b, #0x6
+eor	z5.b, z5.b, #0xf9
+eor3	z29.d, z29.d, z30.d, z31.d
+eorbt	z0.b, z1.b, z31.b
+eorbt	z0.d, z1.d, z31.d
+eorbt	z0.h, z1.h, z31.h
+eorbt	z0.s, z1.s, z31.s
+eors	p0.b, p0/z, p0.b, p1.b
+eortb	z0.b, z1.b, z31.b
+eortb	z0.d, z1.d, z31.d
+eortb	z0.h, z1.h, z31.h
+eortb	z0.s, z1.s, z31.s
+eorv	b0, p7, z31.b
+eorv	d0, p7, z31.d
+eorv	h0, p7, z31.h
+eorv	s0, p7, z31.s
+ext	z0.b, { z1.b, z2.b }, #0
+ext	z31.b, z31.b, z0.b, #0
+ext	z31.b, z31.b, z0.b, #255
+ext	z31.b, { z30.b, z31.b }, #255
+fabd	z0.d, p7/m, z0.d, z31.d
+fabd	z0.h, p7/m, z0.h, z31.h
+fabd	z0.s, p7/m, z0.s, z31.s
+fabs	z31.d, p7/m, z31.d
+fabs	z31.h, p7/m, z31.h
+fabs	z31.s, p7/m, z31.s
+facge	p0.d, p0/z, z0.d, z1.d
+facge	p0.d, p0/z, z1.d, z0.d
+facge	p0.h, p0/z, z0.h, z1.h
+facge	p0.h, p0/z, z1.h, z0.h
+facge	p0.s, p0/z, z0.s, z1.s
+facge	p0.s, p0/z, z1.s, z0.s
+facgt	p0.d, p0/z, z0.d, z1.d
+facgt	p0.d, p0/z, z1.d, z0.d
+facgt	p0.h, p0/z, z0.h, z1.h
+facgt	p0.h, p0/z, z1.h, z0.h
+facgt	p0.s, p0/z, z0.s, z1.s
+facgt	p0.s, p0/z, z1.s, z0.s
+fadd	z0.d, p0/m, z0.d, #0.5
+fadd	z0.d, p7/m, z0.d, z31.d
+fadd	z0.d, z1.d, z31.d
+fadd	z0.h, p0/m, z0.h, #0.5
+fadd	z0.h, p7/m, z0.h, z31.h
+fadd	z0.h, z1.h, z31.h
+fadd	z0.s, p0/m, z0.s, #0.5
+fadd	z0.s, p7/m, z0.s, z31.s
+fadd	z0.s, z1.s, z31.s
+fadd	z31.d, p7/m, z31.d, #1.0
+fadd	z31.h, p7/m, z31.h, #1.0
+fadd	z31.s, p7/m, z31.s, #1.0
+fadda	d0, p7, d0, z31.d
+fadda	h0, p7, h0, z31.h
+fadda	s0, p7, s0, z31.s
+faddp	z0.h, p0/m, z0.h, z1.h
+faddp	z29.s, p3/m, z29.s, z30.s
+faddp	z31.d, p7/m, z31.d, z30.d
+faddv	d0, p7, z31.d
+faddv	h0, p7, z31.h
+faddv	s0, p7, z31.s
+fcadd	z0.d, p0/m, z0.d, z0.d, #90
+fcadd	z0.h, p0/m, z0.h, z0.h, #90
+fcadd	z0.s, p0/m, z0.s, z0.s, #90
+fcadd	z31.d, p7/m, z31.d, z31.d, #270
+fcadd	z31.h, p7/m, z31.h, z31.h, #270
+fcadd	z31.s, p7/m, z31.s, z31.s, #270
+fcmeq	p0.d, p0/z, z0.d, #0.0
+fcmeq	p0.d, p0/z, z0.d, z1.d
+fcmeq	p0.h, p0/z, z0.h, #0.0
+fcmeq	p0.h, p0/z, z0.h, z1.h
+fcmeq	p0.s, p0/z, z0.s, #0.0
+fcmeq	p0.s, p0/z, z0.s, z1.s
+fcmge	p0.d, p0/z, z0.d, #0.0
+fcmge	p0.d, p0/z, z0.d, z1.d
+fcmge	p0.d, p0/z, z1.d, z0.d
+fcmge	p0.h, p0/z, z0.h, #0.0
+fcmge	p0.h, p0/z, z0.h, z1.h
+fcmge	p0.h, p0/z, z1.h, z0.h
+fcmge	p0.s, p0/z, z0.s, #0.0
+fcmge	p0.s, p0/z, z0.s, z1.s
+fcmge	p0.s, p0/z, z1.s, z0.s
+fcmgt	p0.d, p0/z, z0.d, #0.0
+fcmgt	p0.d, p0/z, z0.d, z1.d
+fcmgt	p0.d, p0/z, z1.d, z0.d
+fcmgt	p0.h, p0/z, z0.h, #0.0
+fcmgt	p0.h, p0/z, z0.h, z1.h
+fcmgt	p0.h, p0/z, z1.h, z0.h
+fcmgt	p0.s, p0/z, z0.s, #0.0
+fcmgt	p0.s, p0/z, z0.s, z1.s
+fcmgt	p0.s, p0/z, z1.s, z0.s
+fcmla	z0.d, p0/m, z0.d, z0.d, #0
+fcmla	z0.d, p0/m, z1.d, z2.d, #90
+fcmla	z0.h, p0/m, z0.h, z0.h, #0
+fcmla	z0.h, p0/m, z1.h, z2.h, #90
+fcmla	z0.h, z0.h, z0.h[0], #0
+fcmla	z0.s, p0/m, z0.s, z0.s, #0
+fcmla	z0.s, p0/m, z1.s, z2.s, #90
+fcmla	z21.s, z10.s, z5.s[1], #90
+fcmla	z23.s, z13.s, z8.s[0], #270
+fcmla	z29.d, p7/m, z30.d, z31.d, #180
+fcmla	z29.h, p7/m, z30.h, z31.h, #180
+fcmla	z29.s, p7/m, z30.s, z31.s, #180
+fcmla	z31.d, p7/m, z31.d, z31.d, #270
+fcmla	z31.h, p7/m, z31.h, z31.h, #270
+fcmla	z31.h, z31.h, z7.h[3], #270
+fcmla	z31.s, p7/m, z31.s, z31.s, #270
+fcmle	p0.d, p0/z, z0.d, #0.0
+fcmle	p0.h, p0/z, z0.h, #0.0
+fcmle	p0.s, p0/z, z0.s, #0.0
+fcmlt	p0.d, p0/z, z0.d, #0.0
+fcmlt	p0.h, p0/z, z0.h, #0.0
+fcmlt	p0.s, p0/z, z0.s, #0.0
+fcmne	p0.d, p0/z, z0.d, #0.0
+fcmne	p0.d, p0/z, z0.d, z1.d
+fcmne	p0.h, p0/z, z0.h, #0.0
+fcmne	p0.h, p0/z, z0.h, z1.h
+fcmne	p0.s, p0/z, z0.s, #0.0
+fcmne	p0.s, p0/z, z0.s, z1.s
+fcmuo	p0.d, p0/z, z0.d, z1.d
+fcmuo	p0.h, p0/z, z0.h, z1.h
+fcmuo	p0.s, p0/z, z0.s, z1.s
+fcvt	z0.d, p0/m, z0.h
+fcvt	z0.d, p0/m, z0.s
+fcvt	z0.h, p0/m, z0.d
+fcvt	z0.h, p0/m, z0.s
+fcvt	z0.s, p0/m, z0.d
+fcvt	z0.s, p0/m, z0.h
+fcvtlt	z0.s, p0/m, z1.h
+fcvtlt	z30.d, p7/m, z31.s
+fcvtnt	z0.h, p0/m, z1.s
+fcvtnt	z30.s, p7/m, z31.d
+fcvtx	z0.s, p0/m, z0.d
+fcvtx	z30.s, p7/m, z31.d
+fcvtxnt	z0.s, p0/m, z1.d
+fcvtxnt	z30.s, p7/m, z31.d
+fcvtzs	z0.d, p0/m, z0.d
+fcvtzs	z0.d, p0/m, z0.h
+fcvtzs	z0.d, p0/m, z0.s
+fcvtzs	z0.h, p0/m, z0.h
+fcvtzs	z0.s, p0/m, z0.d
+fcvtzs	z0.s, p0/m, z0.h
+fcvtzs	z0.s, p0/m, z0.s
+fcvtzu	z0.d, p0/m, z0.d
+fcvtzu	z0.d, p0/m, z0.h
+fcvtzu	z0.d, p0/m, z0.s
+fcvtzu	z0.h, p0/m, z0.h
+fcvtzu	z0.s, p0/m, z0.d
+fcvtzu	z0.s, p0/m, z0.h
+fcvtzu	z0.s, p0/m, z0.s
+fdiv	z0.d, p7/m, z0.d, z31.d
+fdiv	z0.h, p7/m, z0.h, z31.h
+fdiv	z0.s, p7/m, z0.s, z31.s
+fdivr	z0.d, p7/m, z0.d, z31.d
+fdivr	z0.h, p7/m, z0.h, z31.h
+fdivr	z0.s, p7/m, z0.s, z31.s
+fexpa	z0.d, z31.d
+fexpa	z0.h, z31.h
+fexpa	z0.s, z31.s
+flogb	z31.d, p7/m, z31.d
+flogb	z31.h, p7/m, z31.h
+flogb	z31.s, p7/m, z31.s
+fmad	z0.d, p7/m, z1.d, z31.d
+fmad	z0.h, p7/m, z1.h, z31.h
+fmad	z0.s, p7/m, z1.s, z31.s
+fmax	z0.d, p0/m, z0.d, #0.0
+fmax	z0.d, p7/m, z0.d, z31.d
+fmax	z0.h, p0/m, z0.h, #0.0
+fmax	z0.h, p7/m, z0.h, z31.h
+fmax	z0.s, p0/m, z0.s, #0.0
+fmax	z0.s, p7/m, z0.s, z31.s
+fmax	z31.d, p7/m, z31.d, #1.0
+fmax	z31.h, p7/m, z31.h, #1.0
+fmax	z31.s, p7/m, z31.s, #1.0
+fmaxnm	z0.d, p0/m, z0.d, #0.0
+fmaxnm	z0.d, p7/m, z0.d, z31.d
+fmaxnm	z0.h, p0/m, z0.h, #0.0
+fmaxnm	z0.h, p7/m, z0.h, z31.h
+fmaxnm	z0.s, p0/m, z0.s, #0.0
+fmaxnm	z0.s, p7/m, z0.s, z31.s
+fmaxnm	z31.d, p7/m, z31.d, #1.0
+fmaxnm	z31.h, p7/m, z31.h, #1.0
+fmaxnm	z31.s, p7/m, z31.s, #1.0
+fmaxnmp	z0.h, p0/m, z0.h, z1.h
+fmaxnmp	z29.s, p3/m, z29.s, z30.s
+fmaxnmp	z31.d, p7/m, z31.d, z30.d
+fmaxnmv	d0, p7, z31.d
+fmaxnmv	h0, p7, z31.h
+fmaxnmv	s0, p7, z31.s
+fmaxp	z0.h, p0/m, z0.h, z1.h
+fmaxp	z29.s, p3/m, z29.s, z30.s
+fmaxp	z31.d, p7/m, z31.d, z30.d
+fmaxv	d0, p7, z31.d
+fmaxv	h0, p7, z31.h
+fmaxv	s0, p7, z31.s
+fmin	z0.d, p0/m, z0.d, #0.0
+fmin	z0.d, p7/m, z0.d, z31.d
+fmin	z0.h, p0/m, z0.h, #0.0
+fmin	z0.h, p7/m, z0.h, z31.h
+fmin	z0.s, p0/m, z0.s, #0.0
+fmin	z0.s, p7/m, z0.s, z31.s
+fmin	z31.d, p7/m, z31.d, #1.0
+fmin	z31.h, p7/m, z31.h, #1.0
+fmin	z31.s, p7/m, z31.s, #1.0
+fminnm	z0.d, p0/m, z0.d, #0.0
+fminnm	z0.d, p7/m, z0.d, z31.d
+fminnm	z0.h, p0/m, z0.h, #0.0
+fminnm	z0.h, p7/m, z0.h, z31.h
+fminnm	z0.s, p0/m, z0.s, #0.0
+fminnm	z0.s, p7/m, z0.s, z31.s
+fminnm	z31.d, p7/m, z31.d, #1.0
+fminnm	z31.h, p7/m, z31.h, #1.0
+fminnm	z31.s, p7/m, z31.s, #1.0
+fminnmp	z0.h, p0/m, z0.h, z1.h
+fminnmp	z29.s, p3/m, z29.s, z30.s
+fminnmp	z31.d, p7/m, z31.d, z30.d
+fminnmv	d0, p7, z31.d
+fminnmv	h0, p7, z31.h
+fminnmv	s0, p7, z31.s
+fminp	z0.h, p0/m, z0.h, z1.h
+fminp	z29.s, p3/m, z29.s, z30.s
+fminp	z31.d, p7/m, z31.d, z30.d
+fminv	d0, p7, z31.d
+fminv	h0, p7, z31.h
+fminv	s0, p7, z31.s
+fmla	z0.d, p7/m, z1.d, z31.d
+fmla	z0.d, z1.d, z7.d[1]
+fmla	z0.h, p7/m, z1.h, z31.h
+fmla	z0.h, z1.h, z7.h[7]
+fmla	z0.s, p7/m, z1.s, z31.s
+fmla	z0.s, z1.s, z7.s[3]
+fmlalb	z0.s, z1.h, z7.h[0]
+fmlalb	z29.s, z30.h, z31.h
+fmlalb	z30.s, z31.h, z7.h[7]
+fmlalt	z0.s, z1.h, z7.h[0]
+fmlalt	z29.s, z30.h, z31.h
+fmlalt	z30.s, z31.h, z7.h[7]
+fmls	z0.d, p7/m, z1.d, z31.d
+fmls	z0.d, z1.d, z7.d[1]
+fmls	z0.h, p7/m, z1.h, z31.h
+fmls	z0.h, z1.h, z7.h[7]
+fmls	z0.s, p7/m, z1.s, z31.s
+fmls	z0.s, z1.s, z7.s[3]
+fmlslb	z0.s, z1.h, z7.h[0]
+fmlslb	z29.s, z30.h, z31.h
+fmlslb	z30.s, z31.h, z7.h[7]
+fmlslt	z0.s, z1.h, z7.h[0]
+fmlslt	z29.s, z30.h, z31.h
+fmlslt	z30.s, z31.h, z7.h[7]
+fmov	z0.d, #-10.00000000
+fmov	z0.d, #0.12500000
+fmov	z0.d, p0/m, #-10.00000000
+fmov	z0.d, p0/m, #0.12500000
+fmov	z0.h, #-0.12500000
+fmov	z0.h, p0/m, #-0.12500000
+fmov	z0.s, #-0.12500000
+fmov	z0.s, p0/m, #-0.12500000
+fmsb	z0.d, p7/m, z1.d, z31.d
+fmsb	z0.h, p7/m, z1.h, z31.h
+fmsb	z0.s, p7/m, z1.s, z31.s
+fmul	z0.d, p0/m, z0.d, #0.5
+fmul	z0.d, p7/m, z0.d, z31.d
+fmul	z0.d, z0.d, z0.d[0]
+fmul	z0.d, z1.d, z31.d
+fmul	z0.h, p0/m, z0.h, #0.5
+fmul	z0.h, p7/m, z0.h, z31.h
+fmul	z0.h, z0.h, z0.h[0]
+fmul	z0.h, z1.h, z31.h
+fmul	z0.s, p0/m, z0.s, #0.5
+fmul	z0.s, p7/m, z0.s, z31.s
+fmul	z0.s, z0.s, z0.s[0]
+fmul	z0.s, z1.s, z31.s
+fmul	z31.d, p7/m, z31.d, #2.0
+fmul	z31.d, z31.d, z15.d[1]
+fmul	z31.h, p7/m, z31.h, #2.0
+fmul	z31.h, z31.h, z7.h[7]
+fmul	z31.s, p7/m, z31.s, #2.0
+fmul	z31.s, z31.s, z7.s[3]
+fmulx	z0.d, p7/m, z0.d, z31.d
+fmulx	z0.h, p7/m, z0.h, z31.h
+fmulx	z0.s, p7/m, z0.s, z31.s
+fneg	z31.d, p7/m, z31.d
+fneg	z31.h, p7/m, z31.h
+fneg	z31.s, p7/m, z31.s
+fnmad	z0.d, p7/m, z1.d, z31.d
+fnmad	z0.h, p7/m, z1.h, z31.h
+fnmad	z0.s, p7/m, z1.s, z31.s
+fnmla	z0.d, p7/m, z1.d, z31.d
+fnmla	z0.h, p7/m, z1.h, z31.h
+fnmla	z0.s, p7/m, z1.s, z31.s
+fnmls	z0.d, p7/m, z1.d, z31.d
+fnmls	z0.h, p7/m, z1.h, z31.h
+fnmls	z0.s, p7/m, z1.s, z31.s
+fnmsb	z0.d, p7/m, z1.d, z31.d
+fnmsb	z0.h, p7/m, z1.h, z31.h
+fnmsb	z0.s, p7/m, z1.s, z31.s
+frecpe	z0.d, z31.d
+frecpe	z0.h, z31.h
+frecpe	z0.s, z31.s
+frecps	z0.d, z1.d, z31.d
+frecps	z0.h, z1.h, z31.h
+frecps	z0.s, z1.s, z31.s
+frecpx	z31.d, p7/m, z31.d
+frecpx	z31.h, p7/m, z31.h
+frecpx	z31.s, p7/m, z31.s
+frinta	z31.d, p7/m, z31.d
+frinta	z31.h, p7/m, z31.h
+frinta	z31.s, p7/m, z31.s
+frinti	z31.d, p7/m, z31.d
+frinti	z31.h, p7/m, z31.h
+frinti	z31.s, p7/m, z31.s
+frintm	z31.d, p7/m, z31.d
+frintm	z31.h, p7/m, z31.h
+frintm	z31.s, p7/m, z31.s
+frintn	z31.d, p7/m, z31.d
+frintn	z31.h, p7/m, z31.h
+frintn	z31.s, p7/m, z31.s
+frintp	z31.d, p7/m, z31.d
+frintp	z31.h, p7/m, z31.h
+frintp	z31.s, p7/m, z31.s
+frintx	z31.d, p7/m, z31.d
+frintx	z31.h, p7/m, z31.h
+frintx	z31.s, p7/m, z31.s
+frintz	z31.d, p7/m, z31.d
+frintz	z31.h, p7/m, z31.h
+frintz	z31.s, p7/m, z31.s
+frsqrte	z0.d, z31.d
+frsqrte	z0.h, z31.h
+frsqrte	z0.s, z31.s
+frsqrts	z0.d, z1.d, z31.d
+frsqrts	z0.h, z1.h, z31.h
+frsqrts	z0.s, z1.s, z31.s
+fscale	z0.d, p7/m, z0.d, z31.d
+fscale	z0.h, p7/m, z0.h, z31.h
+fscale	z0.s, p7/m, z0.s, z31.s
+fsqrt	z31.d, p7/m, z31.d
+fsqrt	z31.h, p7/m, z31.h
+fsqrt	z31.s, p7/m, z31.s
+fsub	z0.d, p0/m, z0.d, #0.5
+fsub	z0.d, p7/m, z0.d, z31.d
+fsub	z0.d, z1.d, z31.d
+fsub	z0.h, p0/m, z0.h, #0.5
+fsub	z0.h, p7/m, z0.h, z31.h
+fsub	z0.h, z1.h, z31.h
+fsub	z0.s, p0/m, z0.s, #0.5
+fsub	z0.s, p7/m, z0.s, z31.s
+fsub	z0.s, z1.s, z31.s
+fsub	z31.d, p7/m, z31.d, #1.0
+fsub	z31.h, p7/m, z31.h, #1.0
+fsub	z31.s, p7/m, z31.s, #1.0
+fsubr	z0.d, p0/m, z0.d, #0.5
+fsubr	z0.d, p7/m, z0.d, z31.d
+fsubr	z0.h, p0/m, z0.h, #0.5
+fsubr	z0.h, p7/m, z0.h, z31.h
+fsubr	z0.s, p0/m, z0.s, #0.5
+fsubr	z0.s, p7/m, z0.s, z31.s
+fsubr	z31.d, p7/m, z31.d, #1.0
+fsubr	z31.h, p7/m, z31.h, #1.0
+fsubr	z31.s, p7/m, z31.s, #1.0
+ftmad	z0.d, z0.d, z31.d, #7
+ftmad	z0.h, z0.h, z31.h, #7
+ftmad	z0.s, z0.s, z31.s, #7
+ftsmul	z0.d, z1.d, z31.d
+ftsmul	z0.h, z1.h, z31.h
+ftsmul	z0.s, z1.s, z31.s
+ftssel	z0.d, z1.d, z31.d
+ftssel	z0.h, z1.h, z31.h
+ftssel	z0.s, z1.s, z31.s
+histcnt	z0.s, p0/z, z1.s, z2.s
+histcnt	z29.d, p7/z, z30.d, z31.d
+histseg	z0.b, z1.b, z31.b
+incb	x0
+incb	x0, #14
+incb	x0, all, mul #16
+incb	x0, pow2
+incb	x0, vl1
+incd	x0
+incd	x0, #14
+incd	x0, all, mul #16
+incd	x0, pow2
+incd	x0, vl1
+incd	z0.d
+incd	z0.d, all, mul #16
+inch	x0
+inch	x0, #14
+inch	x0, all, mul #16
+inch	x0, pow2
+inch	x0, vl1
+inch	z0.h
+inch	z0.h, all, mul #16
+incp	x0, p0.b
+incp	x0, p0.d
+incp	x0, p0.h
+incp	x0, p0.s
+incp	xzr, p15.b
+incp	xzr, p15.d
+incp	xzr, p15.h
+incp	xzr, p15.s
+incp	z31.d, p15.d
+incp	z31.h, p15.h
+incp	z31.s, p15.s
+incw	x0
+incw	x0, #14
+incw	x0, all, mul #16
+incw	x0, pow2
+incw	x0, vl1
+incw	z0.s
+incw	z0.s, all, mul #16
+index	z0.b, #0, #0
+index	z0.d, #0, #0
+index	z0.h, #0, #0
+index	z0.h, w0, w0
+index	z0.s, #0, #0
+index	z21.b, w10, w21
+index	z21.d, x10, x21
+index	z21.s, w10, w21
+index	z23.b, #13, w8
+index	z23.b, w13, #8
+index	z23.d, #13, x8
+index	z23.d, x13, #8
+index	z23.h, #13, w8
+index	z23.h, w13, #8
+index	z23.s, #13, w8
+index	z23.s, w13, #8
+index	z31.b, #-1, #-1
+index	z31.b, #-1, wzr
+index	z31.b, wzr, #-1
+index	z31.b, wzr, wzr
+index	z31.d, #-1, #-1
+index	z31.d, #-1, xzr
+index	z31.d, xzr, #-1
+index	z31.d, xzr, xzr
+index	z31.h, #-1, #-1
+index	z31.h, #-1, wzr
+index	z31.h, wzr, #-1
+index	z31.h, wzr, wzr
+index	z31.s, #-1, #-1
+index	z31.s, #-1, wzr
+index	z31.s, wzr, #-1
+index	z31.s, wzr, wzr
+insr	z0.b, w0
+insr	z0.d, x0
+insr	z0.h, w0
+insr	z0.s, w0
+insr	z31.b, b31
+insr	z31.b, wzr
+insr	z31.d, d31
+insr	z31.d, xzr
+insr	z31.h, h31
+insr	z31.h, wzr
+insr	z31.s, s31
+insr	z31.s, wzr
+lasta	b0, p7, z31.b
+lasta	d0, p7, z31.d
+lasta	h0, p7, z31.h
+lasta	s0, p7, z31.s
+lasta	w0, p7, z31.b
+lasta	w0, p7, z31.h
+lasta	w0, p7, z31.s
+lasta	x0, p7, z31.d
+lastb	b0, p7, z31.b
+lastb	d0, p7, z31.d
+lastb	h0, p7, z31.h
+lastb	s0, p7, z31.s
+lastb	w0, p7, z31.b
+lastb	w0, p7, z31.h
+lastb	w0, p7, z31.s
+lastb	x0, p7, z31.d
+ld1b	{ z0.b }, p0/z, [sp, x0]
+ld1b	{ z0.b }, p0/z, [x0, x0]
+ld1b	{ z0.b }, p0/z, [x0]
+ld1b	{ z0.d }, p0/z, [x0]
+ld1b	{ z0.d }, p0/z, [z0.d]
+ld1b	{ z0.h }, p0/z, [x0]
+ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1b	{ z0.s }, p0/z, [x0]
+ld1b	{ z0.s }, p0/z, [z0.s]
+ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.s }, p5/z, [x10, x21]
+ld1b	{ z23.d }, p3/z, [x13, x8]
+ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.d }, p7/z, [sp, z31.d]
+ld1b	{ z31.d }, p7/z, [z31.d, #31]
+ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.s }, p7/z, [z31.s, #31]
+ld1b	{ z5.h }, p3/z, [x17, x16]
+ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+ld1d	{ z0.d }, p0/z, [x0]
+ld1d	{ z0.d }, p0/z, [z0.d]
+ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1d	{ z31.d }, p7/z, [sp, z31.d]
+ld1d	{ z31.d }, p7/z, [z31.d, #248]
+ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ld1h	{ z0.d }, p0/z, [x0]
+ld1h	{ z0.d }, p0/z, [z0.d]
+ld1h	{ z0.h }, p0/z, [x0]
+ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1h	{ z0.s }, p0/z, [x0]
+ld1h	{ z0.s }, p0/z, [z0.s]
+ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.d }, p7/z, [sp, z31.d]
+ld1h	{ z31.d }, p7/z, [z31.d, #62]
+ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ld1h	{ z31.s }, p7/z, [z31.s, #62]
+ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+ld1rb	{ z0.b }, p0/z, [x0]
+ld1rb	{ z0.d }, p0/z, [x0]
+ld1rb	{ z0.h }, p0/z, [x0]
+ld1rb	{ z0.s }, p0/z, [x0]
+ld1rb	{ z31.b }, p7/z, [sp, #63]
+ld1rb	{ z31.d }, p7/z, [sp, #63]
+ld1rb	{ z31.h }, p7/z, [sp, #63]
+ld1rb	{ z31.s }, p7/z, [sp, #63]
+ld1rd	{ z0.d }, p0/z, [x0]
+ld1rd	{ z31.d }, p7/z, [sp, #504]
+ld1rh	{ z0.d }, p0/z, [x0]
+ld1rh	{ z0.h }, p0/z, [x0]
+ld1rh	{ z0.s }, p0/z, [x0]
+ld1rh	{ z31.d }, p7/z, [sp, #126]
+ld1rh	{ z31.h }, p7/z, [sp, #126]
+ld1rh	{ z31.s }, p7/z, [sp, #126]
+ld1rqb	{ z0.b }, p0/z, [x0, x0]
+ld1rqb	{ z0.b }, p0/z, [x0]
+ld1rqb	{ z21.b }, p5/z, [x10, #112]
+ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ld1rqd	{ z0.d }, p0/z, [x0]
+ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+ld1rqd	{ z23.d }, p3/z, [x13, #112]
+ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ld1rqh	{ z0.h }, p0/z, [x0]
+ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+ld1rqh	{ z23.h }, p3/z, [x13, #112]
+ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ld1rqw	{ z0.s }, p0/z, [x0]
+ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+ld1rqw	{ z23.s }, p3/z, [x13, #112]
+ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+ld1rsb	{ z0.d }, p0/z, [x0]
+ld1rsb	{ z0.h }, p0/z, [x0]
+ld1rsb	{ z0.s }, p0/z, [x0]
+ld1rsb	{ z31.d }, p7/z, [sp, #63]
+ld1rsb	{ z31.h }, p7/z, [sp, #63]
+ld1rsb	{ z31.s }, p7/z, [sp, #63]
+ld1rsh	{ z0.d }, p0/z, [x0]
+ld1rsh	{ z0.s }, p0/z, [x0]
+ld1rsh	{ z31.d }, p7/z, [sp, #126]
+ld1rsh	{ z31.s }, p7/z, [sp, #126]
+ld1rsw	{ z0.d }, p0/z, [x0]
+ld1rsw	{ z31.d }, p7/z, [sp, #252]
+ld1rw	{ z0.d }, p0/z, [x0]
+ld1rw	{ z0.s }, p0/z, [x0]
+ld1rw	{ z31.d }, p7/z, [sp, #252]
+ld1rw	{ z31.s }, p7/z, [sp, #252]
+ld1sb	{ z0.d }, p0/z, [x0]
+ld1sb	{ z0.d }, p0/z, [z0.d]
+ld1sb	{ z0.h }, p0/z, [sp, x0]
+ld1sb	{ z0.h }, p0/z, [x0, x0]
+ld1sb	{ z0.h }, p0/z, [x0]
+ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1sb	{ z0.s }, p0/z, [x0]
+ld1sb	{ z0.s }, p0/z, [z0.s]
+ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.s }, p5/z, [x10, x21]
+ld1sb	{ z23.d }, p3/z, [x13, x8]
+ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ld1sh	{ z0.d }, p0/z, [x0]
+ld1sh	{ z0.d }, p0/z, [z0.d]
+ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1sh	{ z0.s }, p0/z, [x0]
+ld1sh	{ z0.s }, p0/z, [z0.s]
+ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ld1sw	{ z0.d }, p0/z, [x0]
+ld1sw	{ z0.d }, p0/z, [z0.d]
+ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ld1w	{ z0.d }, p0/z, [x0]
+ld1w	{ z0.d }, p0/z, [z0.d]
+ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1w	{ z0.s }, p0/z, [x0]
+ld1w	{ z0.s }, p0/z, [z0.s]
+ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1w	{ z31.d }, p7/z, [sp, z31.d]
+ld1w	{ z31.d }, p7/z, [z31.d, #124]
+ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+ld1w	{ z31.s }, p7/z, [z31.s, #124]
+ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+ld2b	{ z0.b, z1.b }, p0/z, [x0]
+ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+ld2d	{ z0.d, z1.d }, p0/z, [x0]
+ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+ld2h	{ z0.h, z1.h }, p0/z, [x0]
+ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+ld2w	{ z0.s, z1.s }, p0/z, [x0]
+ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+ld3b	{ z0.b, z1.b, z2.b }, p0/z, [x0, x0]
+ld3b	{ z0.b, z1.b, z2.b }, p0/z, [x0]
+ld3b	{ z21.b, z22.b, z23.b }, p5/z, [x10, #15, mul vl]
+ld3b	{ z23.b, z24.b, z25.b }, p3/z, [x13, #-24, mul vl]
+ld3b	{ z5.b, z6.b, z7.b }, p3/z, [x17, x16]
+ld3d	{ z0.d, z1.d, z2.d }, p0/z, [x0, x0, lsl #3]
+ld3d	{ z0.d, z1.d, z2.d }, p0/z, [x0]
+ld3d	{ z21.d, z22.d, z23.d }, p5/z, [x10, #15, mul vl]
+ld3d	{ z23.d, z24.d, z25.d }, p3/z, [x13, #-24, mul vl]
+ld3d	{ z5.d, z6.d, z7.d }, p3/z, [x17, x16, lsl #3]
+ld3h	{ z0.h, z1.h, z2.h }, p0/z, [x0, x0, lsl #1]
+ld3h	{ z0.h, z1.h, z2.h }, p0/z, [x0]
+ld3h	{ z21.h, z22.h, z23.h }, p5/z, [x10, #15, mul vl]
+ld3h	{ z23.h, z24.h, z25.h }, p3/z, [x13, #-24, mul vl]
+ld3h	{ z5.h, z6.h, z7.h }, p3/z, [x17, x16, lsl #1]
+ld3w	{ z0.s, z1.s, z2.s }, p0/z, [x0, x0, lsl #2]
+ld3w	{ z0.s, z1.s, z2.s }, p0/z, [x0]
+ld3w	{ z21.s, z22.s, z23.s }, p5/z, [x10, #15, mul vl]
+ld3w	{ z23.s, z24.s, z25.s }, p3/z, [x13, #-24, mul vl]
+ld3w	{ z5.s, z6.s, z7.s }, p3/z, [x17, x16, lsl #2]
+ld4b	{ z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x0]
+ld4b	{ z0.b, z1.b, z2.b, z3.b }, p0/z, [x0]
+ld4b	{ z21.b, z22.b, z23.b, z24.b }, p5/z, [x10, #20, mul vl]
+ld4b	{ z23.b, z24.b, z25.b, z26.b }, p3/z, [x13, #-32, mul vl]
+ld4b	{ z5.b, z6.b, z7.b, z8.b }, p3/z, [x17, x16]
+ld4d	{ z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, x0, lsl #3]
+ld4d	{ z0.d, z1.d, z2.d, z3.d }, p0/z, [x0]
+ld4d	{ z21.d, z22.d, z23.d, z24.d }, p5/z, [x10, #20, mul vl]
+ld4d	{ z23.d, z24.d, z25.d, z26.d }, p3/z, [x13, #-32, mul vl]
+ld4d	{ z5.d, z6.d, z7.d, z8.d }, p3/z, [x17, x16, lsl #3]
+ld4h	{ z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, x0, lsl #1]
+ld4h	{ z0.h, z1.h, z2.h, z3.h }, p0/z, [x0]
+ld4h	{ z21.h, z22.h, z23.h, z24.h }, p5/z, [x10, #20, mul vl]
+ld4h	{ z23.h, z24.h, z25.h, z26.h }, p3/z, [x13, #-32, mul vl]
+ld4h	{ z5.h, z6.h, z7.h, z8.h }, p3/z, [x17, x16, lsl #1]
+ld4w	{ z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, x0, lsl #2]
+ld4w	{ z0.s, z1.s, z2.s, z3.s }, p0/z, [x0]
+ld4w	{ z21.s, z22.s, z23.s, z24.s }, p5/z, [x10, #20, mul vl]
+ld4w	{ z23.s, z24.s, z25.s, z26.s }, p3/z, [x13, #-32, mul vl]
+ld4w	{ z5.s, z6.s, z7.s, z8.s }, p3/z, [x17, x16, lsl #2]
+ldff1b	{ z0.d }, p0/z, [x0, x0]
+ldff1b	{ z0.d }, p0/z, [z0.d]
+ldff1b	{ z0.h }, p0/z, [x0, x0]
+ldff1b	{ z0.s }, p0/z, [x0, x0]
+ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1b	{ z0.s }, p0/z, [z0.s]
+ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1b	{ z31.b }, p7/z, [sp]
+ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+ldff1b	{ z31.d }, p7/z, [sp]
+ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+ldff1b	{ z31.h }, p7/z, [sp]
+ldff1b	{ z31.s }, p7/z, [sp]
+ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+ldff1d	{ z0.d }, p0/z, [z0.d]
+ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+ldff1d	{ z31.d }, p7/z, [sp]
+ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ldff1h	{ z0.d }, p0/z, [z0.d]
+ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1h	{ z0.s }, p0/z, [z0.s]
+ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+ldff1h	{ z31.d }, p7/z, [sp]
+ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+ldff1h	{ z31.h }, p7/z, [sp]
+ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ldff1h	{ z31.s }, p7/z, [sp]
+ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+ldff1sb	{ z0.d }, p0/z, [x0, x0]
+ldff1sb	{ z0.d }, p0/z, [z0.d]
+ldff1sb	{ z0.h }, p0/z, [x0, x0]
+ldff1sb	{ z0.s }, p0/z, [x0, x0]
+ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1sb	{ z0.s }, p0/z, [z0.s]
+ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sb	{ z31.d }, p7/z, [sp]
+ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+ldff1sb	{ z31.h }, p7/z, [sp]
+ldff1sb	{ z31.s }, p7/z, [sp]
+ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ldff1sh	{ z0.d }, p0/z, [z0.d]
+ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1sh	{ z0.s }, p0/z, [z0.s]
+ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sh	{ z31.d }, p7/z, [sp]
+ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ldff1sh	{ z31.s }, p7/z, [sp]
+ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ldff1sw	{ z0.d }, p0/z, [z0.d]
+ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sw	{ z31.d }, p7/z, [sp]
+ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ldff1w	{ z0.d }, p0/z, [z0.d]
+ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1w	{ z0.s }, p0/z, [z0.s]
+ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+ldff1w	{ z31.d }, p7/z, [sp]
+ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+ldff1w	{ z31.s }, p7/z, [sp]
+ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+ldnf1b	{ z0.b }, p0/z, [x0]
+ldnf1b	{ z0.d }, p0/z, [x0]
+ldnf1b	{ z0.h }, p0/z, [x0]
+ldnf1b	{ z0.s }, p0/z, [x0]
+ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1d	{ z0.d }, p0/z, [x0]
+ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z0.d }, p0/z, [x0]
+ldnf1h	{ z0.h }, p0/z, [x0]
+ldnf1h	{ z0.s }, p0/z, [x0]
+ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z0.d }, p0/z, [x0]
+ldnf1sb	{ z0.h }, p0/z, [x0]
+ldnf1sb	{ z0.s }, p0/z, [x0]
+ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sh	{ z0.d }, p0/z, [x0]
+ldnf1sh	{ z0.s }, p0/z, [x0]
+ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sw	{ z0.d }, p0/z, [x0]
+ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1w	{ z0.d }, p0/z, [x0]
+ldnf1w	{ z0.s }, p0/z, [x0]
+ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnt1b	{ z0.b }, p0/z, [x0, x0]
+ldnt1b	{ z0.b }, p0/z, [x0]
+ldnt1b	{ z0.d }, p0/z, [z1.d]
+ldnt1b	{ z0.s }, p0/z, [z1.s]
+ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1b	{ z31.d }, p7/z, [z31.d]
+ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1b	{ z31.s }, p7/z, [z31.s]
+ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ldnt1d	{ z0.d }, p0/z, [x0]
+ldnt1d	{ z0.d }, p0/z, [z1.d]
+ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1d	{ z31.d }, p7/z, [z31.d]
+ldnt1h	{ z0.d }, p0/z, [z1.d]
+ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ldnt1h	{ z0.h }, p0/z, [x0]
+ldnt1h	{ z0.s }, p0/z, [z1.s]
+ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1h	{ z31.d }, p7/z, [z31.d]
+ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1h	{ z31.s }, p7/z, [z31.s]
+ldnt1sb	{ z0.d }, p0/z, [z1.d]
+ldnt1sb	{ z0.s }, p0/z, [z1.s]
+ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sb	{ z31.d }, p7/z, [z31.d]
+ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1sb	{ z31.s }, p7/z, [z31.s]
+ldnt1sh	{ z0.d }, p0/z, [z1.d]
+ldnt1sh	{ z0.s }, p0/z, [z1.s]
+ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sh	{ z31.d }, p7/z, [z31.d]
+ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1sh	{ z31.s }, p7/z, [z31.s]
+ldnt1sw	{ z0.d }, p0/z, [z1.d]
+ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sw	{ z31.d }, p7/z, [z31.d]
+ldnt1w	{ z0.d }, p0/z, [z1.d]
+ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ldnt1w	{ z0.s }, p0/z, [x0]
+ldnt1w	{ z0.s }, p0/z, [z1.s]
+ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1w	{ z31.d }, p7/z, [z31.d]
+ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1w	{ z31.s }, p7/z, [z31.s]
+ldr	p0, [x0]
+ldr	p5, [x10, #255, mul vl]
+ldr	p7, [x13, #-256, mul vl]
+ldr	z0, [x0]
+ldr	z23, [x13, #255, mul vl]
+ldr	z31, [sp, #-256, mul vl]
+lsl	z0.b, p0/m, z0.b, #0
+lsl	z0.b, p0/m, z0.b, z0.b
+lsl	z0.b, p0/m, z0.b, z1.d
+lsl	z0.b, z0.b, #0
+lsl	z0.b, z1.b, z2.d
+lsl	z0.d, p0/m, z0.d, #0
+lsl	z0.d, p0/m, z0.d, z0.d
+lsl	z0.d, z0.d, #0
+lsl	z0.h, p0/m, z0.h, #0
+lsl	z0.h, p0/m, z0.h, z0.h
+lsl	z0.h, p0/m, z0.h, z1.d
+lsl	z0.h, z0.h, #0
+lsl	z0.h, z1.h, z2.d
+lsl	z0.s, p0/m, z0.s, #0
+lsl	z0.s, p0/m, z0.s, z0.s
+lsl	z0.s, p0/m, z0.s, z1.d
+lsl	z0.s, z0.s, #0
+lsl	z0.s, z1.s, z2.d
+lsl	z31.b, p0/m, z31.b, #7
+lsl	z31.b, z31.b, #7
+lsl	z31.d, p0/m, z31.d, #63
+lsl	z31.d, z31.d, #63
+lsl	z31.h, p0/m, z31.h, #15
+lsl	z31.h, z31.h, #15
+lsl	z31.s, p0/m, z31.s, #31
+lsl	z31.s, z31.s, #31
+lslr	z0.b, p0/m, z0.b, z0.b
+lslr	z0.d, p0/m, z0.d, z0.d
+lslr	z0.h, p0/m, z0.h, z0.h
+lslr	z0.s, p0/m, z0.s, z0.s
+lsr	z0.b, p0/m, z0.b, #1
+lsr	z0.b, p0/m, z0.b, z0.b
+lsr	z0.b, p0/m, z0.b, z1.d
+lsr	z0.b, z0.b, #1
+lsr	z0.b, z1.b, z2.d
+lsr	z0.d, p0/m, z0.d, #1
+lsr	z0.d, p0/m, z0.d, z0.d
+lsr	z0.d, z0.d, #1
+lsr	z0.h, p0/m, z0.h, #1
+lsr	z0.h, p0/m, z0.h, z0.h
+lsr	z0.h, p0/m, z0.h, z1.d
+lsr	z0.h, z0.h, #1
+lsr	z0.h, z1.h, z2.d
+lsr	z0.s, p0/m, z0.s, #1
+lsr	z0.s, p0/m, z0.s, z0.s
+lsr	z0.s, p0/m, z0.s, z1.d
+lsr	z0.s, z0.s, #1
+lsr	z0.s, z1.s, z2.d
+lsr	z31.b, p0/m, z31.b, #8
+lsr	z31.b, z31.b, #8
+lsr	z31.d, p0/m, z31.d, #64
+lsr	z31.d, z31.d, #64
+lsr	z31.h, p0/m, z31.h, #16
+lsr	z31.h, z31.h, #16
+lsr	z31.s, p0/m, z31.s, #32
+lsr	z31.s, z31.s, #32
+lsrr	z0.b, p0/m, z0.b, z0.b
+lsrr	z0.d, p0/m, z0.d, z0.d
+lsrr	z0.h, p0/m, z0.h, z0.h
+lsrr	z0.s, p0/m, z0.s, z0.s
+mad	z0.b, p7/m, z1.b, z31.b
+mad	z0.d, p7/m, z1.d, z31.d
+mad	z0.h, p7/m, z1.h, z31.h
+mad	z0.s, p7/m, z1.s, z31.s
+match	p0.b, p0/z, z0.b, z0.b
+match	p0.h, p0/z, z0.h, z0.h
+match	p15.b, p7/z, z30.b, z31.b
+match	p15.h, p7/z, z30.h, z31.h
+mla	z0.b, p7/m, z1.b, z31.b
+mla	z0.d, p7/m, z1.d, z31.d
+mla	z0.d, z1.d, z7.d[1]
+mla	z0.h, p7/m, z1.h, z31.h
+mla	z0.h, z1.h, z7.h[7]
+mla	z0.s, p7/m, z1.s, z31.s
+mla	z0.s, z1.s, z7.s[3]
+mls	z0.b, p7/m, z1.b, z31.b
+mls	z0.d, p7/m, z1.d, z31.d
+mls	z0.d, z1.d, z7.d[1]
+mls	z0.h, p7/m, z1.h, z31.h
+mls	z0.h, z1.h, z7.h[7]
+mls	z0.s, p7/m, z1.s, z31.s
+mls	z0.s, z1.s, z7.s[3]
+mov	p0.b, p0.b
+mov	p0.b, p0/m, p0.b
+mov	p0.b, p0/z, p0.b
+mov	p15.b, p15.b
+mov	p15.b, p15/m, p15.b
+mov	p15.b, p15/z, p15.b
+mov	z0.b, #127
+mov	z0.b, b0
+mov	z0.b, p0/m, b0
+mov	z0.b, p0/m, w0
+mov	z0.b, p0/z, #127
+mov	z0.b, w0
+mov	z0.d, #0
+mov	z0.d, #0xe0000000000003ff
+mov	z0.d, #0xffffffffffff7fff
+mov	z0.d, #32768
+mov	z0.d, d0
+mov	z0.d, p0/m, d0
+mov	z0.d, p0/m, x0
+mov	z0.d, x0
+mov	z0.d, z0.d
+mov	z0.h, #-256
+mov	z0.h, #-32768
+mov	z0.h, #0
+mov	z0.h, #32512
+mov	z0.h, #32767
+mov	z0.h, h0
+mov	z0.h, p0/m, h0
+mov	z0.h, p0/m, w0
+mov	z0.h, p0/z, #32512
+mov	z0.h, w0
+mov	z0.q, q0
+mov	z0.s, #0
+mov	z0.s, #0xffff7fff
+mov	z0.s, #32768
+mov	z0.s, p0/m, s0
+mov	z0.s, p0/m, w0
+mov	z0.s, s0
+mov	z0.s, w0
+mov	z21.d, #-128
+mov	z21.d, #-32768
+mov	z21.d, #127
+mov	z21.d, #32512
+mov	z21.d, p0/z, #-128
+mov	z21.d, p0/z, #-32768
+mov	z21.d, p0/z, #127
+mov	z21.d, p0/z, #32512
+mov	z21.d, p15/m, #-128
+mov	z21.d, p15/m, #-32768
+mov	z21.h, #-128
+mov	z21.h, #-32768
+mov	z21.h, #127
+mov	z21.h, #32512
+mov	z21.h, p0/z, #-128
+mov	z21.h, p0/z, #-32768
+mov	z21.h, p0/z, #127
+mov	z21.h, p0/z, #32512
+mov	z21.h, p15/m, #-128
+mov	z21.h, p15/m, #-32768
+mov	z21.s, #-128
+mov	z21.s, #-32768
+mov	z21.s, #127
+mov	z21.s, #32512
+mov	z21.s, p0/z, #-128
+mov	z21.s, p0/z, #-32768
+mov	z21.s, p0/z, #127
+mov	z21.s, p0/z, #32512
+mov	z21.s, p15/m, #-128
+mov	z21.s, p15/m, #-32768
+mov	z31.b, p15/m, z31.b
+mov	z31.b, p7/m, b31
+movprfx z31, z6
+mov	z31.b, p7/m, wsp
+mov	z31.b, wsp
+mov	z31.b, z31.b[63]
+mov	z31.d, p15/m, z31.d
+mov	z31.d, p7/m, d31
+movprfx z31.d, p7/z, z6.d
+mov	z31.d, p7/m, sp
+mov	z31.d, sp
+mov	z31.d, z0.d
+mov	z31.d, z31.d[7]
+mov	z31.h, p15/m, z31.h
+mov	z31.h, p7/m, h31
+mov	z31.h, p7/m, wsp
+mov	z31.h, wsp
+mov	z31.h, z31.h[31]
+mov	z31.s, p15/m, z31.s
+mov	z31.s, p7/m, s31
+mov	z31.s, p7/m, wsp
+mov	z31.s, wsp
+mov	z31.s, z31.s[15]
+mov	z5.b, #-1
+mov	z5.b, #-128
+mov	z5.b, #127
+mov	z5.b, p0/z, #-1
+mov	z5.b, p0/z, #-128
+mov	z5.b, p0/z, #127
+mov	z5.b, p15/m, #-128
+mov	z5.d, #-6
+mov	z5.h, #-6
+mov	z5.q, z17.q[3]
+mov	z5.s, #-6
+movs	p0.b, p0.b
+movs	p0.b, p0/z, p0.b
+movs	p15.b, p15.b
+movs	p15.b, p15/z, p15.b
+mrs	x3, ID_AA64ZFR0_EL1
+mrs	x3, ZCR_EL1
+mrs	x3, ZCR_EL12
+mrs	x3, ZCR_EL2
+mrs	x3, ZCR_EL3
+msb	z0.b, p7/m, z1.b, z31.b
+msb	z0.d, p7/m, z1.d, z31.d
+msb	z0.h, p7/m, z1.h, z31.h
+msb	z0.s, p7/m, z1.s, z31.s
+msr	ZCR_EL1, x3
+msr	ZCR_EL12, x3
+msr	ZCR_EL2, x3
+msr	ZCR_EL3, x3
+mul	z0.b, p7/m, z0.b, z31.b
+mul	z0.b, z1.b, z2.b
+mul	z0.d, p7/m, z0.d, z31.d
+mul	z0.d, z1.d, z15.d[1]
+mul	z0.h, p7/m, z0.h, z31.h
+mul	z0.h, z1.h, z2.h
+mul	z0.h, z1.h, z7.h[7]
+mul	z0.s, p7/m, z0.s, z31.s
+mul	z0.s, z1.s, z7.s[3]
+mul	z29.s, z30.s, z31.s
+mul	z31.b, z31.b, #-128
+mul	z31.b, z31.b, #127
+mul	z31.d, z31.d, #-128
+mul	z31.d, z31.d, #127
+mul	z31.d, z31.d, z31.d
+mul	z31.h, z31.h, #-128
+mul	z31.h, z31.h, #127
+mul	z31.s, z31.s, #-128
+mul	z31.s, z31.s, #127
+nand	p0.b, p0/z, p0.b, p0.b
+nand	p15.b, p15/z, p15.b, p15.b
+nands	p0.b, p0/z, p0.b, p0.b
+nands	p15.b, p15/z, p15.b, p15.b
+nbsl	z0.d, z0.d, z1.d, z2.d
+neg	z0.b, p0/m, z0.b
+neg	z0.d, p0/m, z0.d
+neg	z0.h, p0/m, z0.h
+neg	z0.s, p0/m, z0.s
+neg	z31.b, p7/m, z31.b
+neg	z31.d, p7/m, z31.d
+neg	z31.h, p7/m, z31.h
+neg	z31.s, p7/m, z31.s
+nmatch	p0.b, p0/z, z0.b, z0.b
+nmatch	p0.h, p0/z, z0.h, z0.h
+nmatch	p15.b, p7/z, z30.b, z31.b
+nmatch	p15.h, p7/z, z30.h, z31.h
+nor	p0.b, p0/z, p0.b, p0.b
+nor	p15.b, p15/z, p15.b, p15.b
+nors	p0.b, p0/z, p0.b, p0.b
+nors	p15.b, p15/z, p15.b, p15.b
+not	p0.b, p0/z, p0.b
+not	p15.b, p15/z, p15.b
+not	z31.b, p7/m, z31.b
+not	z31.d, p7/m, z31.d
+not	z31.h, p7/m, z31.h
+not	z31.s, p7/m, z31.s
+nots	p0.b, p0/z, p0.b
+nots	p15.b, p15/z, p15.b
+orn	p0.b, p0/z, p0.b, p0.b
+orn	p15.b, p15/z, p15.b, p15.b
+orns	p0.b, p0/z, p0.b, p0.b
+orns	p15.b, p15/z, p15.b, p15.b
+orr	p0.b, p0/z, p0.b, p1.b
+orr	z0.d, z0.d, #0x6
+orr	z0.d, z0.d, #0xfffffffffffffff9
+orr	z0.s, z0.s, #0x6
+orr	z0.s, z0.s, #0xfffffff9
+orr	z23.d, z13.d, z8.d
+orr	z23.h, z23.h, #0x6
+orr	z23.h, z23.h, #0xfff9
+orr	z31.b, p7/m, z31.b, z31.b
+orr	z31.d, p7/m, z31.d, z31.d
+orr	z31.h, p7/m, z31.h, z31.h
+orr	z31.s, p7/m, z31.s, z31.s
+orr	z5.b, z5.b, #0x6
+orr	z5.b, z5.b, #0xf9
+orrs	p0.b, p0/z, p0.b, p1.b
+orv	b0, p7, z31.b
+orv	d0, p7, z31.d
+orv	h0, p7, z31.h
+orv	s0, p7, z31.s
+pfalse	p15.b
+pfirst	p0.b, p15, p0.b
+pfirst	p15.b, p15, p15.b
+pmul	z0.b, z1.b, z2.b
+pmul	z29.b, z30.b, z31.b
+pmullb	z0.h, z1.b, z2.b
+pmullb	z29.q, z30.d, z31.d
+pmullb	z31.d, z31.s, z31.s
+pmullt	z0.h, z1.b, z2.b
+pmullt	z29.q, z30.d, z31.d
+pmullt	z31.d, z31.s, z31.s
+pnext	p0.b, p15, p0.b
+pnext	p0.d, p15, p0.d
+pnext	p0.h, p15, p0.h
+pnext	p0.s, p15, p0.s
+pnext	p15.b, p15, p15.b
+prfb	#14, p0, [x0]
+prfb	#15, p0, [x0]
+prfb	#6, p0, [x0]
+prfb	#7, p0, [x0]
+prfb	#7, p3, [z13.s, #31]
+prfb	#7, p3, [z13.s]
+prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+prfb	pldl1keep, p0, [x0, z0.d]
+prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+prfb	pldl1keep, p0, [x0]
+prfb	pldl1strm, p0, [x0, #-32, mul vl]
+prfb	pldl1strm, p0, [x0, #31, mul vl]
+prfb	pldl1strm, p0, [x0]
+prfb	pldl2keep, p0, [x0]
+prfb	pldl2strm, p0, [x0]
+prfb	pldl3keep, p0, [x0]
+prfb	pldl3strm, p0, [x0]
+prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+prfb	pldl3strm, p5, [z10.d, #31]
+prfb	pldl3strm, p5, [z10.d]
+prfb	pstl1keep, p0, [x0]
+prfb	pstl1strm, p0, [x0]
+prfb	pstl2keep, p0, [x0]
+prfb	pstl2strm, p0, [x0]
+prfb	pstl3keep, p0, [x0]
+prfb	pstl3strm, p0, [x0]
+prfd	#14, p0, [x0]
+prfd	#15, p0, [x0]
+prfd	#15, p7, [z31.d, #248]
+prfd	#15, p7, [z31.d]
+prfd	#15, p7, [z31.s, #248]
+prfd	#15, p7, [z31.s]
+prfd	#6, p0, [x0]
+prfd	#7, p0, [x0]
+prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+prfd	pldl1keep, p0, [x0]
+prfd	pldl1strm, p0, [x0, #-32, mul vl]
+prfd	pldl1strm, p0, [x0, #31, mul vl]
+prfd	pldl1strm, p0, [x0]
+prfd	pldl2keep, p0, [x0]
+prfd	pldl2strm, p0, [x0]
+prfd	pldl3keep, p0, [x0]
+prfd	pldl3strm, p0, [x0]
+prfd	pstl1keep, p0, [x0]
+prfd	pstl1strm, p0, [x0]
+prfd	pstl2keep, p0, [x0]
+prfd	pstl2strm, p0, [x0]
+prfd	pstl3keep, p0, [x0]
+prfd	pstl3strm, p0, [x0]
+prfh	#14, p0, [x0]
+prfh	#15, p0, [x0]
+prfh	#15, p7, [z31.d, #62]
+prfh	#15, p7, [z31.d]
+prfh	#15, p7, [z31.s, #62]
+prfh	#15, p7, [z31.s]
+prfh	#6, p0, [x0]
+prfh	#7, p0, [x0]
+prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+prfh	pldl1keep, p0, [x0]
+prfh	pldl1strm, p0, [x0, #-32, mul vl]
+prfh	pldl1strm, p0, [x0, #31, mul vl]
+prfh	pldl1strm, p0, [x0]
+prfh	pldl2keep, p0, [x0]
+prfh	pldl2strm, p0, [x0]
+prfh	pldl3keep, p0, [x0]
+prfh	pldl3strm, p0, [x0]
+prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+prfh	pstl1keep, p0, [x0]
+prfh	pstl1strm, p0, [x0]
+prfh	pstl2keep, p0, [x0]
+prfh	pstl2strm, p0, [x0]
+prfh	pstl3keep, p0, [x0]
+prfh	pstl3strm, p0, [x0]
+prfw	#14, p0, [x0]
+prfw	#15, p0, [x0]
+prfw	#15, p7, [z31.d, #124]
+prfw	#15, p7, [z31.d]
+prfw	#15, p7, [z31.s, #124]
+prfw	#15, p7, [z31.s]
+prfw	#6, p0, [x0]
+prfw	#7, p0, [x0]
+prfw	#7, p3, [x13, z8.d, uxtw #2]
+prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+prfw	pldl1keep, p0, [x0]
+prfw	pldl1strm, p0, [x0, #-32, mul vl]
+prfw	pldl1strm, p0, [x0, #31, mul vl]
+prfw	pldl1strm, p0, [x0]
+prfw	pldl2keep, p0, [x0]
+prfw	pldl2strm, p0, [x0]
+prfw	pldl3keep, p0, [x0]
+prfw	pldl3strm, p0, [x0]
+prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+prfw	pstl1keep, p0, [x0]
+prfw	pstl1strm, p0, [x0]
+prfw	pstl2keep, p0, [x0]
+prfw	pstl2strm, p0, [x0]
+prfw	pstl3keep, p0, [x0]
+prfw	pstl3strm, p0, [x0]
+ptest	p15, p0.b
+ptest	p15, p15.b
+ptrue	p0.b, pow2
+ptrue	p0.d, pow2
+ptrue	p0.h, pow2
+ptrue	p0.s, pow2
+ptrue	p15.b
+ptrue	p15.d
+ptrue	p15.h
+ptrue	p15.s
+ptrue	p7.s
+ptrue	p7.s, #14
+ptrue	p7.s, #15
+ptrue	p7.s, #16
+ptrue	p7.s, #17
+ptrue	p7.s, #18
+ptrue	p7.s, #19
+ptrue	p7.s, #20
+ptrue	p7.s, #21
+ptrue	p7.s, #22
+ptrue	p7.s, #23
+ptrue	p7.s, #24
+ptrue	p7.s, #25
+ptrue	p7.s, #26
+ptrue	p7.s, #27
+ptrue	p7.s, #28
+ptrue	p7.s, mul3
+ptrue	p7.s, mul4
+ptrue	p7.s, vl1
+ptrue	p7.s, vl128
+ptrue	p7.s, vl16
+ptrue	p7.s, vl2
+ptrue	p7.s, vl256
+ptrue	p7.s, vl3
+ptrue	p7.s, vl32
+ptrue	p7.s, vl4
+ptrue	p7.s, vl5
+ptrue	p7.s, vl6
+ptrue	p7.s, vl64
+ptrue	p7.s, vl7
+ptrue	p7.s, vl8
+ptrues	p0.b, pow2
+ptrues	p0.d, pow2
+ptrues	p0.h, pow2
+ptrues	p0.s, pow2
+ptrues	p15.b
+ptrues	p15.d
+ptrues	p15.h
+ptrues	p15.s
+ptrues	p7.s
+ptrues	p7.s, #14
+ptrues	p7.s, #15
+ptrues	p7.s, #16
+ptrues	p7.s, #17
+ptrues	p7.s, #18
+ptrues	p7.s, #19
+ptrues	p7.s, #20
+ptrues	p7.s, #21
+ptrues	p7.s, #22
+ptrues	p7.s, #23
+ptrues	p7.s, #24
+ptrues	p7.s, #25
+ptrues	p7.s, #26
+ptrues	p7.s, #27
+ptrues	p7.s, #28
+ptrues	p7.s, mul3
+ptrues	p7.s, mul4
+ptrues	p7.s, vl1
+ptrues	p7.s, vl128
+ptrues	p7.s, vl16
+ptrues	p7.s, vl2
+ptrues	p7.s, vl256
+ptrues	p7.s, vl3
+ptrues	p7.s, vl32
+ptrues	p7.s, vl4
+ptrues	p7.s, vl5
+ptrues	p7.s, vl6
+ptrues	p7.s, vl64
+ptrues	p7.s, vl7
+ptrues	p7.s, vl8
+punpkhi	p0.h, p0.b
+punpkhi	p15.h, p15.b
+punpklo	p0.h, p0.b
+punpklo	p15.h, p15.b
+raddhnb	z0.b, z1.h, z31.h
+raddhnb	z0.h, z1.s, z31.s
+raddhnb	z0.s, z1.d, z31.d
+raddhnt	z0.b, z1.h, z31.h
+raddhnt	z0.h, z1.s, z31.s
+raddhnt	z0.s, z1.d, z31.d
+rax1	z0.d, z1.d, z31.d
+rbit	z0.b, p7/m, z31.b
+rbit	z0.d, p7/m, z31.d
+rbit	z0.h, p7/m, z31.h
+rbit	z0.s, p7/m, z31.s
+rdffr	p0.b
+rdffr	p0.b, p0/z
+rdffr	p15.b
+rdffr	p15.b, p15/z
+rdffrs	p0.b, p0/z
+rdffrs	p15.b, p15/z
+rdvl	x0, #0
+rdvl	x21, #-32
+rdvl	x23, #31
+rdvl	xzr, #-1
+rev	z0.b, z31.b
+rev	z0.d, z31.d
+rev	z0.h, z31.h
+rev	z0.s, z31.s
+revb	z0.d, p7/m, z31.d
+revb	z0.h, p7/m, z31.h
+revb	z0.s, p7/m, z31.s
+revh	z0.d, p7/m, z31.d
+revh	z0.s, p7/m, z31.s
+revw	z0.d, p7/m, z31.d
+rshrnb	z0.b, z0.h, #1
+rshrnb	z0.h, z0.s, #1
+rshrnb	z0.s, z0.d, #1
+rshrnb	z31.b, z31.h, #8
+rshrnb	z31.h, z31.s, #16
+rshrnb	z31.s, z31.d, #32
+rshrnt	z0.b, z0.h, #1
+rshrnt	z0.h, z0.s, #1
+rshrnt	z0.s, z0.d, #1
+rshrnt	z31.b, z31.h, #8
+rshrnt	z31.h, z31.s, #16
+rshrnt	z31.s, z31.d, #32
+rsubhnb	z0.b, z1.h, z31.h
+rsubhnb	z0.h, z1.s, z31.s
+rsubhnb	z0.s, z1.d, z31.d
+rsubhnt	z0.b, z1.h, z31.h
+rsubhnt	z0.h, z1.s, z31.s
+rsubhnt	z0.s, z1.d, z31.d
+saba	z0.b, z1.b, z31.b
+saba	z0.d, z1.d, z31.d
+saba	z0.h, z1.h, z31.h
+saba	z0.s, z1.s, z31.s
+sabalb	z0.d, z1.s, z31.s
+sabalb	z0.h, z1.b, z31.b
+sabalb	z0.s, z1.h, z31.h
+sabalt	z0.d, z1.s, z31.s
+sabalt	z0.h, z1.b, z31.b
+sabalt	z0.s, z1.h, z31.h
+sabd	z31.b, p7/m, z31.b, z31.b
+sabd	z31.d, p7/m, z31.d, z31.d
+sabd	z31.h, p7/m, z31.h, z31.h
+sabd	z31.s, p7/m, z31.s, z31.s
+sabdlb	z0.h, z1.b, z2.b
+sabdlb	z29.s, z30.h, z31.h
+sabdlb	z31.d, z31.s, z31.s
+sabdlt	z0.h, z1.b, z2.b
+sabdlt	z29.s, z30.h, z31.h
+sabdlt	z31.d, z31.s, z31.s
+sadalp	z0.h, p0/m, z1.b
+sadalp	z29.s, p0/m, z30.h
+sadalp	z30.d, p7/m, z31.s
+saddlb	z0.h, z1.b, z2.b
+saddlb	z29.s, z30.h, z31.h
+saddlb	z31.d, z31.s, z31.s
+saddlbt	z0.d, z1.s, z31.s
+saddlbt	z0.h, z1.b, z31.b
+saddlbt	z0.s, z1.h, z31.h
+saddlt	z0.h, z1.b, z2.b
+saddlt	z29.s, z30.h, z31.h
+saddlt	z31.d, z31.s, z31.s
+saddv	d0, p7, z31.b
+saddv	d0, p7, z31.h
+saddv	d0, p7, z31.s
+saddwb	z0.h, z1.h, z2.b
+saddwb	z29.s, z30.s, z31.h
+saddwb	z31.d, z31.d, z31.s
+saddwt	z0.h, z1.h, z2.b
+saddwt	z29.s, z30.s, z31.h
+saddwt	z31.d, z31.d, z31.s
+sbclb	z0.d, z1.d, z31.d
+sbclb	z0.s, z1.s, z31.s
+sbclt	z0.d, z1.d, z31.d
+sbclt	z0.s, z1.s, z31.s
+scvtf	z0.d, p0/m, z0.d
+scvtf	z0.d, p0/m, z0.s
+scvtf	z0.h, p0/m, z0.d
+scvtf	z0.h, p0/m, z0.h
+scvtf	z0.h, p0/m, z0.s
+scvtf	z0.s, p0/m, z0.d
+scvtf	z0.s, p0/m, z0.s
+sdiv	z0.d, p7/m, z0.d, z31.d
+sdiv	z0.s, p7/m, z0.s, z31.s
+sdivr	z0.d, p7/m, z0.d, z31.d
+sdivr	z0.s, p7/m, z0.s, z31.s
+sdot	z0.d, z1.h, z15.h[1]
+sdot	z0.d, z1.h, z31.h
+sdot	z0.s, z1.b, z31.b
+sdot	z0.s, z1.b, z7.b[3]
+sel	z23.b, p11, z13.b, z8.b
+sel	z23.d, p11, z13.d, z8.d
+sel	z23.h, p11, z13.h, z8.h
+sel	z23.s, p11, z13.s, z8.s
+setffr
+shadd	z0.b, p0/m, z0.b, z1.b
+shadd	z0.h, p0/m, z0.h, z1.h
+shadd	z29.s, p7/m, z29.s, z30.s
+shadd	z31.d, p7/m, z31.d, z30.d
+shrnb	z0.b, z0.h, #1
+shrnb	z0.h, z0.s, #1
+shrnb	z0.s, z0.d, #1
+shrnb	z31.b, z31.h, #8
+shrnb	z31.h, z31.s, #16
+shrnb	z31.s, z31.d, #32
+shrnt	z0.b, z0.h, #1
+shrnt	z0.h, z0.s, #1
+shrnt	z0.s, z0.d, #1
+shrnt	z31.b, z31.h, #8
+shrnt	z31.h, z31.s, #16
+shrnt	z31.s, z31.d, #32
+shsub	z0.b, p0/m, z0.b, z1.b
+shsub	z0.h, p0/m, z0.h, z1.h
+shsub	z29.s, p7/m, z29.s, z30.s
+shsub	z31.d, p7/m, z31.d, z30.d
+shsubr	z0.b, p0/m, z0.b, z1.b
+shsubr	z0.h, p0/m, z0.h, z1.h
+shsubr	z29.s, p7/m, z29.s, z30.s
+shsubr	z31.d, p7/m, z31.d, z30.d
+sli	z0.b, z0.b, #0
+sli	z0.d, z0.d, #0
+sli	z0.h, z0.h, #0
+sli	z0.s, z0.s, #0
+sli	z31.b, z31.b, #7
+sli	z31.d, z31.d, #63
+sli	z31.h, z31.h, #15
+sli	z31.s, z31.s, #31
+sm4e	z0.s, z0.s, z31.s
+sm4ekey	z0.s, z1.s, z31.s
+smax	z0.b, z0.b, #-128
+smax	z0.d, z0.d, #-128
+smax	z0.h, z0.h, #-128
+smax	z0.s, z0.s, #-128
+smax	z31.b, p7/m, z31.b, z31.b
+smax	z31.b, z31.b, #127
+smax	z31.d, p7/m, z31.d, z31.d
+smax	z31.d, z31.d, #127
+smax	z31.h, p7/m, z31.h, z31.h
+smax	z31.h, z31.h, #127
+smax	z31.s, p7/m, z31.s, z31.s
+smax	z31.s, z31.s, #127
+smaxp	z0.b, p0/m, z0.b, z1.b
+smaxp	z0.h, p0/m, z0.h, z1.h
+smaxp	z29.s, p7/m, z29.s, z30.s
+smaxp	z31.d, p7/m, z31.d, z30.d
+smaxv	b0, p7, z31.b
+smaxv	d0, p7, z31.d
+smaxv	h0, p7, z31.h
+smaxv	s0, p7, z31.s
+smin	z0.b, z0.b, #-128
+smin	z0.d, z0.d, #-128
+smin	z0.h, z0.h, #-128
+smin	z0.s, z0.s, #-128
+smin	z31.b, p7/m, z31.b, z31.b
+smin	z31.b, z31.b, #127
+smin	z31.d, p7/m, z31.d, z31.d
+smin	z31.d, z31.d, #127
+smin	z31.h, p7/m, z31.h, z31.h
+smin	z31.h, z31.h, #127
+smin	z31.s, p7/m, z31.s, z31.s
+smin	z31.s, z31.s, #127
+sminp	z0.b, p0/m, z0.b, z1.b
+sminp	z0.h, p0/m, z0.h, z1.h
+sminp	z29.s, p7/m, z29.s, z30.s
+sminp	z31.d, p7/m, z31.d, z30.d
+sminv	b0, p7, z31.b
+sminv	d0, p7, z31.d
+sminv	h0, p7, z31.h
+sminv	s0, p7, z31.s
+smlalb	z0.d, z1.s, z15.s[1]
+smlalb	z0.d, z1.s, z31.s
+smlalb	z0.h, z1.b, z31.b
+smlalb	z0.s, z1.h, z31.h
+smlalb	z0.s, z1.h, z7.h[7]
+smlalt	z0.d, z1.s, z15.s[1]
+smlalt	z0.d, z1.s, z31.s
+smlalt	z0.h, z1.b, z31.b
+smlalt	z0.s, z1.h, z31.h
+smlalt	z0.s, z1.h, z7.h[7]
+smlslb	z0.d, z1.s, z15.s[1]
+smlslb	z0.d, z1.s, z31.s
+smlslb	z0.h, z1.b, z31.b
+smlslb	z0.s, z1.h, z31.h
+smlslb	z0.s, z1.h, z7.h[7]
+smlslt	z0.d, z1.s, z15.s[1]
+smlslt	z0.d, z1.s, z31.s
+smlslt	z0.h, z1.b, z31.b
+smlslt	z0.s, z1.h, z31.h
+smlslt	z0.s, z1.h, z7.h[7]
+smmla	z0.s, z1.b, z2.b
+smulh	z0.b, p7/m, z0.b, z31.b
+smulh	z0.b, z1.b, z2.b
+smulh	z0.d, p7/m, z0.d, z31.d
+smulh	z0.h, p7/m, z0.h, z31.h
+smulh	z0.h, z1.h, z2.h
+smulh	z0.s, p7/m, z0.s, z31.s
+smulh	z29.s, z30.s, z31.s
+smulh	z31.d, z31.d, z31.d
+smullb	z0.d, z1.s, z15.s[1]
+smullb	z0.h, z1.b, z2.b
+smullb	z0.s, z1.h, z7.h[7]
+smullb	z29.s, z30.h, z31.h
+smullb	z31.d, z31.s, z31.s
+smullt	z0.d, z1.s, z15.s[1]
+smullt	z0.h, z1.b, z2.b
+smullt	z0.s, z1.h, z7.h[7]
+smullt	z29.s, z30.h, z31.h
+smullt	z31.d, z31.s, z31.s
+splice	z29.b, p7, { z30.b, z31.b }
+splice	z29.d, p7, { z30.d, z31.d }
+splice	z29.h, p7, { z30.h, z31.h }
+splice	z29.s, p7, { z30.s, z31.s }
+splice	z31.b, p7, z31.b, z31.b
+splice	z31.d, p7, z31.d, z31.d
+splice	z31.h, p7, z31.h, z31.h
+splice	z31.s, p7, z31.s, z31.s
+sqabs	z31.b, p7/m, z31.b
+sqabs	z31.d, p7/m, z31.d
+sqabs	z31.h, p7/m, z31.h
+sqabs	z31.s, p7/m, z31.s
+sqadd	z0.b, p0/m, z0.b, z1.b
+sqadd	z0.b, z0.b, #0
+sqadd	z0.b, z0.b, z0.b
+sqadd	z0.d, z0.d, #0
+sqadd	z0.d, z0.d, #0, lsl #8
+sqadd	z0.d, z0.d, z0.d
+sqadd	z0.h, p0/m, z0.h, z1.h
+sqadd	z0.h, z0.h, #0
+sqadd	z0.h, z0.h, #0, lsl #8
+sqadd	z0.h, z0.h, z0.h
+sqadd	z0.s, z0.s, #0
+sqadd	z0.s, z0.s, #0, lsl #8
+sqadd	z0.s, z0.s, z0.s
+sqadd	z29.s, p7/m, z29.s, z30.s
+sqadd	z31.b, z31.b, #255
+sqadd	z31.d, p7/m, z31.d, z30.d
+sqadd	z31.d, z31.d, #65280
+sqadd	z31.h, z31.h, #65280
+sqadd	z31.s, z31.s, #65280
+sqcadd	z0.b, z0.b, z0.b, #90
+sqcadd	z0.d, z0.d, z0.d, #90
+sqcadd	z0.h, z0.h, z0.h, #90
+sqcadd	z0.s, z0.s, z0.s, #90
+sqcadd	z31.b, z31.b, z31.b, #270
+sqcadd	z31.d, z31.d, z31.d, #270
+sqcadd	z31.h, z31.h, z31.h, #270
+sqcadd	z31.s, z31.s, z31.s, #270
+sqdecb	x0
+sqdecb	x0, #14
+sqdecb	x0, all, mul #16
+sqdecb	x0, pow2
+sqdecb	x0, vl1
+sqdecb	x0, w0
+sqdecb	x0, w0, all, mul #16
+sqdecb	x0, w0, pow2
+sqdecb	x0, w0, pow2, mul #16
+sqdecd	x0
+sqdecd	x0, #14
+sqdecd	x0, all, mul #16
+sqdecd	x0, pow2
+sqdecd	x0, vl1
+sqdecd	x0, w0
+sqdecd	x0, w0, all, mul #16
+sqdecd	x0, w0, pow2
+sqdecd	x0, w0, pow2, mul #16
+sqdecd	z0.d
+sqdecd	z0.d, all, mul #16
+sqdecd	z0.d, pow2
+sqdecd	z0.d, pow2, mul #16
+sqdech	x0
+sqdech	x0, #14
+sqdech	x0, all, mul #16
+sqdech	x0, pow2
+sqdech	x0, vl1
+sqdech	x0, w0
+sqdech	x0, w0, all, mul #16
+sqdech	x0, w0, pow2
+sqdech	x0, w0, pow2, mul #16
+sqdech	z0.h
+sqdech	z0.h, all, mul #16
+sqdech	z0.h, pow2
+sqdech	z0.h, pow2, mul #16
+sqdecp	x0, p0.b
+sqdecp	x0, p0.d
+sqdecp	x0, p0.h
+sqdecp	x0, p0.s
+sqdecp	xzr, p15.b, wzr
+sqdecp	xzr, p15.d, wzr
+sqdecp	xzr, p15.h, wzr
+sqdecp	xzr, p15.s, wzr
+sqdecp	z0.d, p0.d
+sqdecp	z0.h, p0.h
+sqdecp	z0.s, p0.s
+sqdecw	x0
+sqdecw	x0, #14
+sqdecw	x0, all, mul #16
+sqdecw	x0, pow2
+sqdecw	x0, vl1
+sqdecw	x0, w0
+sqdecw	x0, w0, all, mul #16
+sqdecw	x0, w0, pow2
+sqdecw	x0, w0, pow2, mul #16
+sqdecw	z0.s
+sqdecw	z0.s, all, mul #16
+sqdecw	z0.s, pow2
+sqdecw	z0.s, pow2, mul #16
+sqdmlalb	z0.d, z1.s, z15.s[3]
+sqdmlalb	z0.d, z1.s, z31.s
+sqdmlalb	z0.h, z1.b, z31.b
+sqdmlalb	z0.s, z1.h, z31.h
+sqdmlalb	z0.s, z1.h, z7.h[7]
+sqdmlalbt	z0.d, z1.s, z31.s
+sqdmlalbt	z0.h, z1.b, z31.b
+sqdmlalbt	z0.s, z1.h, z31.h
+sqdmlalt	z0.d, z1.s, z15.s[3]
+sqdmlalt	z0.d, z1.s, z31.s
+sqdmlalt	z0.h, z1.b, z31.b
+sqdmlalt	z0.s, z1.h, z31.h
+sqdmlalt	z0.s, z1.h, z7.h[7]
+sqdmlslb	z0.d, z1.s, z15.s[3]
+sqdmlslb	z0.d, z1.s, z31.s
+sqdmlslb	z0.h, z1.b, z31.b
+sqdmlslb	z0.s, z1.h, z31.h
+sqdmlslb	z0.s, z1.h, z7.h[7]
+sqdmlslbt	z0.d, z1.s, z31.s
+sqdmlslbt	z0.h, z1.b, z31.b
+sqdmlslbt	z0.s, z1.h, z31.h
+sqdmlslt	z0.d, z1.s, z15.s[3]
+sqdmlslt	z0.d, z1.s, z31.s
+sqdmlslt	z0.h, z1.b, z31.b
+sqdmlslt	z0.s, z1.h, z31.h
+sqdmlslt	z0.s, z1.h, z7.h[7]
+sqdmulh	z0.b, z1.b, z2.b
+sqdmulh	z0.d, z1.d, z15.d[1]
+sqdmulh	z0.h, z1.h, z2.h
+sqdmulh	z0.h, z1.h, z7.h[7]
+sqdmulh	z0.s, z1.s, z7.s[3]
+sqdmulh	z29.s, z30.s, z31.s
+sqdmulh	z31.d, z31.d, z31.d
+sqdmullb	z0.d, z1.s, z15.s[1]
+sqdmullb	z0.h, z1.b, z2.b
+sqdmullb	z0.s, z1.h, z7.h[7]
+sqdmullb	z29.s, z30.h, z31.h
+sqdmullb	z31.d, z31.s, z31.s
+sqdmullt	z0.d, z1.s, z15.s[1]
+sqdmullt	z0.h, z1.b, z2.b
+sqdmullt	z0.s, z1.h, z7.h[7]
+sqdmullt	z29.s, z30.h, z31.h
+sqdmullt	z31.d, z31.s, z31.s
+sqincb	x0
+sqincb	x0, #14
+sqincb	x0, all, mul #16
+sqincb	x0, pow2
+sqincb	x0, vl1
+sqincb	x0, w0
+sqincb	x0, w0, all, mul #16
+sqincb	x0, w0, pow2
+sqincb	x0, w0, pow2, mul #16
+sqincd	x0
+sqincd	x0, #14
+sqincd	x0, all, mul #16
+sqincd	x0, pow2
+sqincd	x0, vl1
+sqincd	x0, w0
+sqincd	x0, w0, all, mul #16
+sqincd	x0, w0, pow2
+sqincd	x0, w0, pow2, mul #16
+sqincd	z0.d
+sqincd	z0.d, all, mul #16
+sqincd	z0.d, pow2
+sqincd	z0.d, pow2, mul #16
+sqinch	x0
+sqinch	x0, #14
+sqinch	x0, all, mul #16
+sqinch	x0, pow2
+sqinch	x0, vl1
+sqinch	x0, w0
+sqinch	x0, w0, all, mul #16
+sqinch	x0, w0, pow2
+sqinch	x0, w0, pow2, mul #16
+sqinch	z0.h
+sqinch	z0.h, all, mul #16
+sqinch	z0.h, pow2
+sqinch	z0.h, pow2, mul #16
+sqincp	x0, p0.b
+sqincp	x0, p0.d
+sqincp	x0, p0.h
+sqincp	x0, p0.s
+sqincp	xzr, p15.b, wzr
+sqincp	xzr, p15.d, wzr
+sqincp	xzr, p15.h, wzr
+sqincp	xzr, p15.s, wzr
+sqincp	z0.d, p0.d
+sqincp	z0.h, p0.h
+sqincp	z0.s, p0.s
+sqincw	x0
+sqincw	x0, #14
+sqincw	x0, all, mul #16
+sqincw	x0, pow2
+sqincw	x0, vl1
+sqincw	x0, w0
+sqincw	x0, w0, all, mul #16
+sqincw	x0, w0, pow2
+sqincw	x0, w0, pow2, mul #16
+sqincw	z0.s
+sqincw	z0.s, all, mul #16
+sqincw	z0.s, pow2
+sqincw	z0.s, pow2, mul #16
+sqneg	z31.b, p7/m, z31.b
+sqneg	z31.d, p7/m, z31.d
+sqneg	z31.h, p7/m, z31.h
+sqneg	z31.s, p7/m, z31.s
+sqrdcmlah	z0.b, z1.b, z2.b, #0
+sqrdcmlah	z0.d, z1.d, z2.d, #0
+sqrdcmlah	z0.h, z1.h, z2.h, #0
+sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+sqrdcmlah	z0.s, z1.s, z2.s, #0
+sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+sqrdcmlah	z15.b, z16.b, z17.b, #270
+sqrdcmlah	z15.d, z16.d, z17.d, #270
+sqrdcmlah	z15.h, z16.h, z17.h, #270
+sqrdcmlah	z15.s, z16.s, z17.s, #270
+sqrdcmlah	z29.b, z30.b, z31.b, #90
+sqrdcmlah	z29.d, z30.d, z31.d, #90
+sqrdcmlah	z29.h, z30.h, z31.h, #90
+sqrdcmlah	z29.s, z30.s, z31.s, #90
+sqrdcmlah	z31.b, z31.b, z31.b, #180
+sqrdcmlah	z31.d, z31.d, z31.d, #180
+sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+sqrdcmlah	z31.h, z31.h, z31.h, #180
+sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+sqrdcmlah	z31.s, z31.s, z31.s, #180
+sqrdmlah	z0.b, z1.b, z31.b
+sqrdmlah	z0.d, z1.d, z15.d[1]
+sqrdmlah	z0.d, z1.d, z31.d
+sqrdmlah	z0.h, z1.h, z31.h
+sqrdmlah	z0.h, z1.h, z7.h[7]
+sqrdmlah	z0.s, z1.s, z31.s
+sqrdmlah	z0.s, z1.s, z7.s[3]
+sqrdmlsh	z0.b, z1.b, z31.b
+sqrdmlsh	z0.d, z1.d, z15.d[1]
+sqrdmlsh	z0.d, z1.d, z31.d
+sqrdmlsh	z0.h, z1.h, z31.h
+sqrdmlsh	z0.h, z1.h, z7.h[7]
+sqrdmlsh	z0.s, z1.s, z31.s
+sqrdmlsh	z0.s, z1.s, z7.s[3]
+sqrdmulh	z0.b, z1.b, z2.b
+sqrdmulh	z0.d, z1.d, z15.d[1]
+sqrdmulh	z0.h, z1.h, z2.h
+sqrdmulh	z0.h, z1.h, z7.h[7]
+sqrdmulh	z0.s, z1.s, z7.s[3]
+sqrdmulh	z29.s, z30.s, z31.s
+sqrdmulh	z31.d, z31.d, z31.d
+sqrshl	z0.b, p0/m, z0.b, z1.b
+sqrshl	z0.h, p0/m, z0.h, z1.h
+sqrshl	z29.s, p7/m, z29.s, z30.s
+sqrshl	z31.d, p7/m, z31.d, z30.d
+sqrshlr	z0.b, p0/m, z0.b, z1.b
+sqrshlr	z0.h, p0/m, z0.h, z1.h
+sqrshlr	z29.s, p7/m, z29.s, z30.s
+sqrshlr	z31.d, p7/m, z31.d, z30.d
+sqrshrnb	z0.b, z0.h, #1
+sqrshrnb	z0.h, z0.s, #1
+sqrshrnb	z0.s, z0.d, #1
+sqrshrnb	z31.b, z31.h, #8
+sqrshrnb	z31.h, z31.s, #16
+sqrshrnb	z31.s, z31.d, #32
+sqrshrnt	z0.b, z0.h, #1
+sqrshrnt	z0.h, z0.s, #1
+sqrshrnt	z0.s, z0.d, #1
+sqrshrnt	z31.b, z31.h, #8
+sqrshrnt	z31.h, z31.s, #16
+sqrshrnt	z31.s, z31.d, #32
+sqrshrunb	z0.b, z0.h, #1
+sqrshrunb	z0.h, z0.s, #1
+sqrshrunb	z0.s, z0.d, #1
+sqrshrunb	z31.b, z31.h, #8
+sqrshrunb	z31.h, z31.s, #16
+sqrshrunb	z31.s, z31.d, #32
+sqrshrunt	z0.b, z0.h, #1
+sqrshrunt	z0.h, z0.s, #1
+sqrshrunt	z0.s, z0.d, #1
+sqrshrunt	z31.b, z31.h, #8
+sqrshrunt	z31.h, z31.s, #16
+sqrshrunt	z31.s, z31.d, #32
+sqshl	z0.b, p0/m, z0.b, #0
+sqshl	z0.b, p0/m, z0.b, z1.b
+sqshl	z0.d, p0/m, z0.d, #0
+sqshl	z0.h, p0/m, z0.h, #0
+sqshl	z0.h, p0/m, z0.h, z1.h
+sqshl	z0.s, p0/m, z0.s, #0
+sqshl	z29.s, p7/m, z29.s, z30.s
+sqshl	z31.b, p0/m, z31.b, #7
+sqshl	z31.d, p0/m, z31.d, #63
+sqshl	z31.d, p7/m, z31.d, z30.d
+sqshl	z31.h, p0/m, z31.h, #15
+sqshl	z31.s, p0/m, z31.s, #31
+sqshlr	z0.b, p0/m, z0.b, z1.b
+sqshlr	z0.h, p0/m, z0.h, z1.h
+sqshlr	z29.s, p7/m, z29.s, z30.s
+sqshlr	z31.d, p7/m, z31.d, z30.d
+sqshlu	z0.b, p0/m, z0.b, #0
+sqshlu	z0.d, p0/m, z0.d, #0
+sqshlu	z0.h, p0/m, z0.h, #0
+sqshlu	z0.s, p0/m, z0.s, #0
+sqshlu	z31.b, p0/m, z31.b, #7
+sqshlu	z31.d, p0/m, z31.d, #63
+sqshlu	z31.h, p0/m, z31.h, #15
+sqshlu	z31.s, p0/m, z31.s, #31
+sqshrnb	z0.b, z0.h, #1
+sqshrnb	z0.h, z0.s, #1
+sqshrnb	z0.s, z0.d, #1
+sqshrnb	z31.b, z31.h, #8
+sqshrnb	z31.h, z31.s, #16
+sqshrnb	z31.s, z31.d, #32
+sqshrnt	z0.b, z0.h, #1
+sqshrnt	z0.h, z0.s, #1
+sqshrnt	z0.s, z0.d, #1
+sqshrnt	z31.b, z31.h, #8
+sqshrnt	z31.h, z31.s, #16
+sqshrnt	z31.s, z31.d, #32
+sqshrunb	z0.b, z0.h, #1
+sqshrunb	z0.h, z0.s, #1
+sqshrunb	z0.s, z0.d, #1
+sqshrunb	z31.b, z31.h, #8
+sqshrunb	z31.h, z31.s, #16
+sqshrunb	z31.s, z31.d, #32
+sqshrunt	z0.b, z0.h, #1
+sqshrunt	z0.h, z0.s, #1
+sqshrunt	z0.s, z0.d, #1
+sqshrunt	z31.b, z31.h, #8
+sqshrunt	z31.h, z31.s, #16
+sqshrunt	z31.s, z31.d, #32
+sqsub	z0.b, p0/m, z0.b, z1.b
+sqsub	z0.b, z0.b, #0
+sqsub	z0.b, z0.b, z0.b
+sqsub	z0.d, z0.d, #0
+sqsub	z0.d, z0.d, #0, lsl #8
+sqsub	z0.d, z0.d, z0.d
+sqsub	z0.h, p0/m, z0.h, z1.h
+sqsub	z0.h, z0.h, #0
+sqsub	z0.h, z0.h, #0, lsl #8
+sqsub	z0.h, z0.h, z0.h
+sqsub	z0.s, z0.s, #0
+sqsub	z0.s, z0.s, #0, lsl #8
+sqsub	z0.s, z0.s, z0.s
+sqsub	z29.s, p7/m, z29.s, z30.s
+sqsub	z31.b, z31.b, #255
+sqsub	z31.d, p7/m, z31.d, z30.d
+sqsub	z31.d, z31.d, #65280
+sqsub	z31.h, z31.h, #65280
+sqsub	z31.s, z31.s, #65280
+sqsubr	z0.b, p0/m, z0.b, z1.b
+sqsubr	z0.h, p0/m, z0.h, z1.h
+sqsubr	z29.s, p7/m, z29.s, z30.s
+sqsubr	z31.d, p7/m, z31.d, z30.d
+sqxtnb	z0.b, z31.h
+sqxtnb	z0.h, z31.s
+sqxtnb	z0.s, z31.d
+sqxtnt	z0.b, z31.h
+sqxtnt	z0.h, z31.s
+sqxtnt	z0.s, z31.d
+sqxtunb	z0.b, z31.h
+sqxtunb	z0.h, z31.s
+sqxtunb	z0.s, z31.d
+sqxtunt	z0.b, z31.h
+sqxtunt	z0.h, z31.s
+sqxtunt	z0.s, z31.d
+srhadd	z0.b, p0/m, z0.b, z1.b
+srhadd	z0.h, p0/m, z0.h, z1.h
+srhadd	z29.s, p7/m, z29.s, z30.s
+srhadd	z31.d, p7/m, z31.d, z30.d
+sri	z0.b, z0.b, #1
+sri	z0.d, z0.d, #1
+sri	z0.h, z0.h, #1
+sri	z0.s, z0.s, #1
+sri	z31.b, z31.b, #8
+sri	z31.d, z31.d, #64
+sri	z31.h, z31.h, #16
+sri	z31.s, z31.s, #32
+srshl	z0.b, p0/m, z0.b, z1.b
+srshl	z0.h, p0/m, z0.h, z1.h
+srshl	z29.s, p7/m, z29.s, z30.s
+srshl	z31.d, p7/m, z31.d, z30.d
+srshlr	z0.b, p0/m, z0.b, z1.b
+srshlr	z0.h, p0/m, z0.h, z1.h
+srshlr	z29.s, p7/m, z29.s, z30.s
+srshlr	z31.d, p7/m, z31.d, z30.d
+srshr	z0.b, p0/m, z0.b, #1
+srshr	z0.d, p0/m, z0.d, #1
+srshr	z0.h, p0/m, z0.h, #1
+srshr	z0.s, p0/m, z0.s, #1
+srshr	z31.b, p0/m, z31.b, #8
+srshr	z31.d, p0/m, z31.d, #64
+srshr	z31.h, p0/m, z31.h, #16
+srshr	z31.s, p0/m, z31.s, #32
+srsra	z0.b, z0.b, #1
+srsra	z0.d, z0.d, #1
+srsra	z0.h, z0.h, #1
+srsra	z0.s, z0.s, #1
+srsra	z31.b, z31.b, #8
+srsra	z31.d, z31.d, #64
+srsra	z31.h, z31.h, #16
+srsra	z31.s, z31.s, #32
+sshllb	z0.d, z0.s, #0
+sshllb	z0.h, z0.b, #0
+sshllb	z0.s, z0.h, #0
+sshllb	z31.d, z31.s, #31
+sshllb	z31.h, z31.b, #7
+sshllb	z31.s, z31.h, #15
+sshllt	z0.d, z0.s, #0
+sshllt	z0.h, z0.b, #0
+sshllt	z0.s, z0.h, #0
+sshllt	z31.d, z31.s, #31
+sshllt	z31.h, z31.b, #7
+sshllt	z31.s, z31.h, #15
+ssra	z0.b, z0.b, #1
+ssra	z0.d, z0.d, #1
+ssra	z0.h, z0.h, #1
+ssra	z0.s, z0.s, #1
+ssra	z31.b, z31.b, #8
+ssra	z31.d, z31.d, #64
+ssra	z31.h, z31.h, #16
+ssra	z31.s, z31.s, #32
+ssublb	z0.h, z1.b, z2.b
+ssublb	z29.s, z30.h, z31.h
+ssublb	z31.d, z31.s, z31.s
+ssublbt	z0.d, z1.s, z31.s
+ssublbt	z0.h, z1.b, z31.b
+ssublbt	z0.s, z1.h, z31.h
+ssublt	z0.h, z1.b, z2.b
+ssublt	z29.s, z30.h, z31.h
+ssublt	z31.d, z31.s, z31.s
+ssubltb	z0.d, z1.s, z31.s
+ssubltb	z0.h, z1.b, z31.b
+ssubltb	z0.s, z1.h, z31.h
+ssubwb	z0.h, z1.h, z2.b
+ssubwb	z29.s, z30.s, z31.h
+ssubwb	z31.d, z31.d, z31.s
+ssubwt	z0.h, z1.h, z2.b
+ssubwt	z29.s, z30.s, z31.h
+ssubwt	z31.d, z31.d, z31.s
+st1b	{ z0.b }, p0, [x0, x0]
+st1b	{ z0.b }, p0, [x0]
+st1b	{ z0.d }, p0, [x0, x0]
+st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1b	{ z0.d }, p0, [x0, z0.d]
+st1b	{ z0.d }, p0, [x0]
+st1b	{ z0.d }, p7, [z0.d]
+st1b	{ z0.h }, p0, [x0, x0]
+st1b	{ z0.h }, p0, [x0]
+st1b	{ z0.s }, p0, [x0, x0]
+st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1b	{ z0.s }, p0, [x0]
+st1b	{ z0.s }, p7, [z0.s]
+st1b	{ z21.b }, p5, [x10, #5, mul vl]
+st1b	{ z21.d }, p5, [x10, #5, mul vl]
+st1b	{ z21.h }, p5, [x10, #5, mul vl]
+st1b	{ z21.s }, p5, [x10, #5, mul vl]
+st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+st1b	{ z31.d }, p7, [z31.d, #31]
+st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+st1b	{ z31.s }, p7, [z31.s, #31]
+st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1d	{ z0.d }, p0, [x0, z0.d]
+st1d	{ z0.d }, p0, [x0]
+st1d	{ z0.d }, p7, [z0.d]
+st1d	{ z21.d }, p5, [x10, #5, mul vl]
+st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+st1d	{ z31.d }, p7, [z31.d, #248]
+st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1h	{ z0.d }, p0, [x0, z0.d]
+st1h	{ z0.d }, p0, [x0]
+st1h	{ z0.d }, p7, [z0.d]
+st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+st1h	{ z0.h }, p0, [x0]
+st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1h	{ z0.s }, p0, [x0]
+st1h	{ z0.s }, p7, [z0.s]
+st1h	{ z21.d }, p5, [x10, #5, mul vl]
+st1h	{ z21.h }, p5, [x10, #5, mul vl]
+st1h	{ z21.s }, p5, [x10, #5, mul vl]
+st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+st1h	{ z31.d }, p7, [z31.d, #62]
+st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+st1h	{ z31.s }, p7, [z31.s, #62]
+st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1w	{ z0.d }, p0, [x0, z0.d]
+st1w	{ z0.d }, p0, [x0]
+st1w	{ z0.d }, p7, [z0.d]
+st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1w	{ z0.s }, p0, [x0]
+st1w	{ z0.s }, p7, [z0.s]
+st1w	{ z21.d }, p5, [x10, #5, mul vl]
+st1w	{ z21.s }, p5, [x10, #5, mul vl]
+st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+st1w	{ z31.d }, p7, [z31.d, #124]
+st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+st1w	{ z31.s }, p7, [z31.s, #124]
+st2b	{ z0.b, z1.b }, p0, [x0, x0]
+st2b	{ z0.b, z1.b }, p0, [x0]
+st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+st2b	{ z5.b, z6.b }, p3, [x17, x16]
+st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+st2d	{ z0.d, z1.d }, p0, [x0]
+st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+st2h	{ z0.h, z1.h }, p0, [x0]
+st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+st2w	{ z0.s, z1.s }, p0, [x0]
+st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+st3b	{ z0.b, z1.b, z2.b }, p0, [x0, x0]
+st3b	{ z0.b, z1.b, z2.b }, p0, [x0]
+st3b	{ z21.b, z22.b, z23.b }, p5, [x10, #15, mul vl]
+st3b	{ z23.b, z24.b, z25.b }, p3, [x13, #-24, mul vl]
+st3b	{ z5.b, z6.b, z7.b }, p3, [x17, x16]
+st3d	{ z0.d, z1.d, z2.d }, p0, [x0, x0, lsl #3]
+st3d	{ z0.d, z1.d, z2.d }, p0, [x0]
+st3d	{ z21.d, z22.d, z23.d }, p5, [x10, #15, mul vl]
+st3d	{ z23.d, z24.d, z25.d }, p3, [x13, #-24, mul vl]
+st3d	{ z5.d, z6.d, z7.d }, p3, [x17, x16, lsl #3]
+st3h	{ z0.h, z1.h, z2.h }, p0, [x0, x0, lsl #1]
+st3h	{ z0.h, z1.h, z2.h }, p0, [x0]
+st3h	{ z21.h, z22.h, z23.h }, p5, [x10, #15, mul vl]
+st3h	{ z23.h, z24.h, z25.h }, p3, [x13, #-24, mul vl]
+st3h	{ z5.h, z6.h, z7.h }, p3, [x17, x16, lsl #1]
+st3w	{ z0.s, z1.s, z2.s }, p0, [x0, x0, lsl #2]
+st3w	{ z0.s, z1.s, z2.s }, p0, [x0]
+st3w	{ z21.s, z22.s, z23.s }, p5, [x10, #15, mul vl]
+st3w	{ z23.s, z24.s, z25.s }, p3, [x13, #-24, mul vl]
+st3w	{ z5.s, z6.s, z7.s }, p3, [x17, x16, lsl #2]
+st4b	{ z0.b, z1.b, z2.b, z3.b }, p0, [x0, x0]
+st4b	{ z0.b, z1.b, z2.b, z3.b }, p0, [x0]
+st4b	{ z21.b, z22.b, z23.b, z24.b }, p5, [x10, #20, mul vl]
+st4b	{ z23.b, z24.b, z25.b, z26.b }, p3, [x13, #-32, mul vl]
+st4b	{ z5.b, z6.b, z7.b, z8.b }, p3, [x17, x16]
+st4d	{ z0.d, z1.d, z2.d, z3.d }, p0, [x0, x0, lsl #3]
+st4d	{ z0.d, z1.d, z2.d, z3.d }, p0, [x0]
+st4d	{ z21.d, z22.d, z23.d, z24.d }, p5, [x10, #20, mul vl]
+st4d	{ z23.d, z24.d, z25.d, z26.d }, p3, [x13, #-32, mul vl]
+st4d	{ z5.d, z6.d, z7.d, z8.d }, p3, [x17, x16, lsl #3]
+st4h	{ z0.h, z1.h, z2.h, z3.h }, p0, [x0, x0, lsl #1]
+st4h	{ z0.h, z1.h, z2.h, z3.h }, p0, [x0]
+st4h	{ z21.h, z22.h, z23.h, z24.h }, p5, [x10, #20, mul vl]
+st4h	{ z23.h, z24.h, z25.h, z26.h }, p3, [x13, #-32, mul vl]
+st4h	{ z5.h, z6.h, z7.h, z8.h }, p3, [x17, x16, lsl #1]
+st4w	{ z0.s, z1.s, z2.s, z3.s }, p0, [x0, x0, lsl #2]
+st4w	{ z0.s, z1.s, z2.s, z3.s }, p0, [x0]
+st4w	{ z21.s, z22.s, z23.s, z24.s }, p5, [x10, #20, mul vl]
+st4w	{ z23.s, z24.s, z25.s, z26.s }, p3, [x13, #-32, mul vl]
+st4w	{ z5.s, z6.s, z7.s, z8.s }, p3, [x17, x16, lsl #2]
+stnt1b	{ z0.b }, p0, [x0, x0]
+stnt1b	{ z0.b }, p0, [x0]
+stnt1b	{ z0.d }, p0, [z1.d]
+stnt1b	{ z0.s }, p0, [z1.s]
+stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+stnt1b	{ z31.d }, p7, [z31.d, x0]
+stnt1b	{ z31.d }, p7, [z31.d]
+stnt1b	{ z31.s }, p7, [z31.s, x0]
+stnt1b	{ z31.s }, p7, [z31.s]
+stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+stnt1d	{ z0.d }, p0, [x0]
+stnt1d	{ z0.d }, p0, [z1.d]
+stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+stnt1d	{ z31.d }, p7, [z31.d, x0]
+stnt1d	{ z31.d }, p7, [z31.d]
+stnt1h	{ z0.d }, p0, [z1.d]
+stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+stnt1h	{ z0.h }, p0, [x0]
+stnt1h	{ z0.s }, p0, [z1.s]
+stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+stnt1h	{ z31.d }, p7, [z31.d, x0]
+stnt1h	{ z31.d }, p7, [z31.d]
+stnt1h	{ z31.s }, p7, [z31.s, x0]
+stnt1h	{ z31.s }, p7, [z31.s]
+stnt1w	{ z0.d }, p0, [z1.d]
+stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+stnt1w	{ z0.s }, p0, [x0]
+stnt1w	{ z0.s }, p0, [z1.s]
+stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+stnt1w	{ z31.d }, p7, [z31.d, x0]
+stnt1w	{ z31.d }, p7, [z31.d]
+stnt1w	{ z31.s }, p7, [z31.s, x0]
+stnt1w	{ z31.s }, p7, [z31.s]
+str	p0, [x0]
+str	p15, [sp, #-256, mul vl]
+str	p5, [x10, #255, mul vl]
+str	z0, [x0]
+str	z21, [x10, #-256, mul vl]
+str	z31, [sp, #255, mul vl]
+sub	z0.b, p0/m, z0.b, z0.b
+sub	z0.b, z0.b, #0
+sub	z0.b, z0.b, z0.b
+sub	z0.d, p0/m, z0.d, z0.d
+sub	z0.d, z0.d, #0
+sub	z0.d, z0.d, #0, lsl #8
+sub	z0.d, z0.d, z0.d
+sub	z0.h, p0/m, z0.h, z0.h
+sub	z0.h, z0.h, #0
+sub	z0.h, z0.h, #0, lsl #8
+sub	z0.h, z0.h, z0.h
+sub	z0.s, p0/m, z0.s, z0.s
+sub	z0.s, z0.s, #0
+sub	z0.s, z0.s, #0, lsl #8
+sub	z0.s, z0.s, z0.s
+sub	z21.b, p5/m, z21.b, z10.b
+sub	z21.b, z10.b, z21.b
+sub	z21.d, p5/m, z21.d, z10.d
+sub	z21.d, z10.d, z21.d
+sub	z21.h, p5/m, z21.h, z10.h
+sub	z21.h, z10.h, z21.h
+sub	z21.s, p5/m, z21.s, z10.s
+sub	z21.s, z10.s, z21.s
+sub	z23.b, p3/m, z23.b, z13.b
+sub	z23.b, z13.b, z8.b
+sub	z23.d, p3/m, z23.d, z13.d
+sub	z23.d, z13.d, z8.d
+sub	z23.h, p3/m, z23.h, z13.h
+sub	z23.h, z13.h, z8.h
+sub	z23.s, p3/m, z23.s, z13.s
+sub	z23.s, z13.s, z8.s
+sub	z31.b, p7/m, z31.b, z31.b
+sub	z31.b, z31.b, #255
+sub	z31.b, z31.b, z31.b
+sub	z31.d, p7/m, z31.d, z31.d
+sub	z31.d, z31.d, #65280
+sub	z31.d, z31.d, z31.d
+sub	z31.h, p7/m, z31.h, z31.h
+sub	z31.h, z31.h, #65280
+sub	z31.h, z31.h, z31.h
+sub	z31.s, p7/m, z31.s, z31.s
+sub	z31.s, z31.s, #65280
+sub	z31.s, z31.s, z31.s
+subhnb	z0.b, z1.h, z31.h
+subhnb	z0.h, z1.s, z31.s
+subhnb	z0.s, z1.d, z31.d
+subhnt	z0.b, z1.h, z31.h
+subhnt	z0.h, z1.s, z31.s
+subhnt	z0.s, z1.d, z31.d
+subr	z0.b, p0/m, z0.b, z0.b
+subr	z0.b, z0.b, #0
+subr	z0.d, p0/m, z0.d, z0.d
+subr	z0.d, z0.d, #0
+subr	z0.d, z0.d, #0, lsl #8
+subr	z0.h, p0/m, z0.h, z0.h
+subr	z0.h, z0.h, #0
+subr	z0.h, z0.h, #0, lsl #8
+subr	z0.s, p0/m, z0.s, z0.s
+subr	z0.s, z0.s, #0
+subr	z0.s, z0.s, #0, lsl #8
+subr	z31.b, z31.b, #255
+subr	z31.d, z31.d, #65280
+subr	z31.h, z31.h, #65280
+subr	z31.s, z31.s, #65280
+sunpkhi	z31.d, z31.s
+sunpkhi	z31.h, z31.b
+sunpkhi	z31.s, z31.h
+sunpklo	z31.d, z31.s
+sunpklo	z31.h, z31.b
+sunpklo	z31.s, z31.h
+suqadd	z0.b, p0/m, z0.b, z1.b
+suqadd	z0.h, p0/m, z0.h, z1.h
+suqadd	z29.s, p7/m, z29.s, z30.s
+suqadd	z31.d, p7/m, z31.d, z30.d
+sxtb	z0.d, p0/m, z0.d
+sxtb	z0.h, p0/m, z0.h
+sxtb	z0.s, p0/m, z0.s
+sxtb	z31.d, p7/m, z31.d
+sxtb	z31.h, p7/m, z31.h
+sxtb	z31.s, p7/m, z31.s
+sxth	z0.d, p0/m, z0.d
+sxth	z0.s, p0/m, z0.s
+sxth	z31.d, p7/m, z31.d
+sxth	z31.s, p7/m, z31.s
+sxtw	z0.d, p0/m, z0.d
+sxtw	z31.d, p7/m, z31.d
+tbl	z28.b, { z29.b, z30.b }, z31.b
+tbl	z28.d, { z29.d, z30.d }, z31.d
+tbl	z28.h, { z29.h, z30.h }, z31.h
+tbl	z28.s, { z29.s, z30.s }, z31.s
+tbl	z31.b, { z31.b }, z31.b
+tbl	z31.d, { z31.d }, z31.d
+tbl	z31.h, { z31.h }, z31.h
+tbl	z31.s, { z31.s }, z31.s
+tbx	z31.b, z31.b, z31.b
+tbx	z31.d, z31.d, z31.d
+tbx	z31.h, z31.h, z31.h
+tbx	z31.s, z31.s, z31.s
+trn1	p15.b, p15.b, p15.b
+trn1	p15.d, p15.d, p15.d
+trn1	p15.h, p15.h, p15.h
+trn1	p15.s, p15.s, p15.s
+trn1	z31.b, z31.b, z31.b
+trn1	z31.d, z31.d, z31.d
+trn1	z31.h, z31.h, z31.h
+trn1	z31.s, z31.s, z31.s
+trn2	p15.b, p15.b, p15.b
+trn2	p15.d, p15.d, p15.d
+trn2	p15.h, p15.h, p15.h
+trn2	p15.s, p15.s, p15.s
+trn2	z31.b, z31.b, z31.b
+trn2	z31.d, z31.d, z31.d
+trn2	z31.h, z31.h, z31.h
+trn2	z31.s, z31.s, z31.s
+uaba	z0.b, z1.b, z31.b
+uaba	z0.d, z1.d, z31.d
+uaba	z0.h, z1.h, z31.h
+uaba	z0.s, z1.s, z31.s
+uabalb	z0.d, z1.s, z31.s
+uabalb	z0.h, z1.b, z31.b
+uabalb	z0.s, z1.h, z31.h
+uabalt	z0.d, z1.s, z31.s
+uabalt	z0.h, z1.b, z31.b
+uabalt	z0.s, z1.h, z31.h
+uabd	z31.b, p7/m, z31.b, z31.b
+uabd	z31.d, p7/m, z31.d, z31.d
+uabd	z31.h, p7/m, z31.h, z31.h
+uabd	z31.s, p7/m, z31.s, z31.s
+uabdlb	z0.h, z1.b, z2.b
+uabdlb	z29.s, z30.h, z31.h
+uabdlb	z31.d, z31.s, z31.s
+uabdlt	z0.h, z1.b, z2.b
+uabdlt	z29.s, z30.h, z31.h
+uabdlt	z31.d, z31.s, z31.s
+uadalp	z0.h, p0/m, z1.b
+uadalp	z29.s, p0/m, z30.h
+uadalp	z30.d, p7/m, z31.s
+uaddlb	z0.h, z1.b, z2.b
+uaddlb	z29.s, z30.h, z31.h
+uaddlb	z31.d, z31.s, z31.s
+uaddlt	z0.h, z1.b, z2.b
+uaddlt	z29.s, z30.h, z31.h
+uaddlt	z31.d, z31.s, z31.s
+uaddv	d0, p7, z31.b
+uaddv	d0, p7, z31.d
+uaddv	d0, p7, z31.h
+uaddv	d0, p7, z31.s
+uaddwb	z0.h, z1.h, z2.b
+uaddwb	z29.s, z30.s, z31.h
+uaddwb	z31.d, z31.d, z31.s
+uaddwt	z0.h, z1.h, z2.b
+uaddwt	z29.s, z30.s, z31.h
+uaddwt	z31.d, z31.d, z31.s
+ucvtf	z0.d, p0/m, z0.d
+ucvtf	z0.d, p0/m, z0.s
+ucvtf	z0.h, p0/m, z0.d
+ucvtf	z0.h, p0/m, z0.h
+ucvtf	z0.h, p0/m, z0.s
+ucvtf	z0.s, p0/m, z0.d
+ucvtf	z0.s, p0/m, z0.s
+udiv	z0.d, p7/m, z0.d, z31.d
+udiv	z0.s, p7/m, z0.s, z31.s
+udivr	z0.d, p7/m, z0.d, z31.d
+udivr	z0.s, p7/m, z0.s, z31.s
+udot	z0.d, z1.h, z15.h[1]
+udot	z0.d, z1.h, z31.h
+udot	z0.s, z1.b, z31.b
+udot	z0.s, z1.b, z7.b[3]
+uhadd	z0.b, p0/m, z0.b, z1.b
+uhadd	z0.h, p0/m, z0.h, z1.h
+uhadd	z29.s, p7/m, z29.s, z30.s
+uhadd	z31.d, p7/m, z31.d, z30.d
+uhsub	z0.b, p0/m, z0.b, z1.b
+uhsub	z0.h, p0/m, z0.h, z1.h
+uhsub	z29.s, p7/m, z29.s, z30.s
+uhsub	z31.d, p7/m, z31.d, z30.d
+uhsubr	z0.b, p0/m, z0.b, z1.b
+uhsubr	z0.h, p0/m, z0.h, z1.h
+uhsubr	z29.s, p7/m, z29.s, z30.s
+uhsubr	z31.d, p7/m, z31.d, z30.d
+umax	z0.b, z0.b, #0
+umax	z31.b, p7/m, z31.b, z31.b
+umax	z31.b, z31.b, #255
+umax	z31.d, p7/m, z31.d, z31.d
+umax	z31.h, p7/m, z31.h, z31.h
+umax	z31.s, p7/m, z31.s, z31.s
+umaxp	z0.b, p0/m, z0.b, z1.b
+umaxp	z0.h, p0/m, z0.h, z1.h
+umaxp	z29.s, p7/m, z29.s, z30.s
+umaxp	z31.d, p7/m, z31.d, z30.d
+umaxv	b0, p7, z31.b
+umaxv	d0, p7, z31.d
+umaxv	h0, p7, z31.h
+umaxv	s0, p7, z31.s
+umin	z0.b, z0.b, #0
+umin	z31.b, p7/m, z31.b, z31.b
+umin	z31.b, z31.b, #255
+umin	z31.d, p7/m, z31.d, z31.d
+umin	z31.h, p7/m, z31.h, z31.h
+umin	z31.s, p7/m, z31.s, z31.s
+uminp	z0.b, p0/m, z0.b, z1.b
+uminp	z0.h, p0/m, z0.h, z1.h
+uminp	z29.s, p7/m, z29.s, z30.s
+uminp	z31.d, p7/m, z31.d, z30.d
+uminv	b0, p7, z31.b
+uminv	d0, p7, z31.d
+uminv	h0, p7, z31.h
+uminv	s0, p7, z31.s
+umlalb	z0.d, z1.s, z15.s[1]
+umlalb	z0.d, z1.s, z31.s
+umlalb	z0.h, z1.b, z31.b
+umlalb	z0.s, z1.h, z31.h
+umlalb	z0.s, z1.h, z7.h[7]
+umlalt	z0.d, z1.s, z15.s[1]
+umlalt	z0.d, z1.s, z31.s
+umlalt	z0.h, z1.b, z31.b
+umlalt	z0.s, z1.h, z31.h
+umlalt	z0.s, z1.h, z7.h[7]
+umlslb	z0.d, z1.s, z15.s[1]
+umlslb	z0.d, z1.s, z31.s
+umlslb	z0.h, z1.b, z31.b
+umlslb	z0.s, z1.h, z31.h
+umlslb	z0.s, z1.h, z7.h[7]
+umlslt	z0.d, z1.s, z15.s[1]
+umlslt	z0.d, z1.s, z31.s
+umlslt	z0.h, z1.b, z31.b
+umlslt	z0.s, z1.h, z31.h
+umlslt	z0.s, z1.h, z7.h[7]
+ummla	z0.s, z1.b, z2.b
+umulh	z0.b, p7/m, z0.b, z31.b
+umulh	z0.b, z1.b, z2.b
+umulh	z0.d, p7/m, z0.d, z31.d
+umulh	z0.h, p7/m, z0.h, z31.h
+umulh	z0.h, z1.h, z2.h
+umulh	z0.s, p7/m, z0.s, z31.s
+umulh	z29.s, z30.s, z31.s
+umulh	z31.d, z31.d, z31.d
+umullb	z0.d, z1.s, z15.s[1]
+umullb	z0.h, z1.b, z2.b
+umullb	z0.s, z1.h, z7.h[7]
+umullb	z29.s, z30.h, z31.h
+umullb	z31.d, z31.s, z31.s
+umullt	z0.d, z1.s, z15.s[1]
+umullt	z0.h, z1.b, z2.b
+umullt	z0.s, z1.h, z7.h[7]
+umullt	z29.s, z30.h, z31.h
+umullt	z31.d, z31.s, z31.s
+uqadd	z0.b, p0/m, z0.b, z1.b
+uqadd	z0.b, z0.b, #0
+uqadd	z0.b, z0.b, z0.b
+uqadd	z0.d, z0.d, #0
+uqadd	z0.d, z0.d, #0, lsl #8
+uqadd	z0.d, z0.d, z0.d
+uqadd	z0.h, p0/m, z0.h, z1.h
+uqadd	z0.h, z0.h, #0
+uqadd	z0.h, z0.h, #0, lsl #8
+uqadd	z0.h, z0.h, z0.h
+uqadd	z0.s, z0.s, #0
+uqadd	z0.s, z0.s, #0, lsl #8
+uqadd	z0.s, z0.s, z0.s
+uqadd	z29.s, p7/m, z29.s, z30.s
+uqadd	z31.b, z31.b, #255
+uqadd	z31.d, p7/m, z31.d, z30.d
+uqadd	z31.d, z31.d, #65280
+uqadd	z31.h, z31.h, #65280
+uqadd	z31.s, z31.s, #65280
+uqdecb	w0
+uqdecb	w0, all, mul #16
+uqdecb	w0, pow2
+uqdecb	w0, pow2, mul #16
+uqdecb	x0
+uqdecb	x0, #14
+uqdecb	x0, all, mul #16
+uqdecb	x0, pow2
+uqdecb	x0, vl1
+uqdecd	w0
+uqdecd	w0, all, mul #16
+uqdecd	w0, pow2
+uqdecd	w0, pow2, mul #16
+uqdecd	x0
+uqdecd	x0, #14
+uqdecd	x0, all, mul #16
+uqdecd	x0, pow2
+uqdecd	x0, vl1
+uqdecd	z0.d
+uqdecd	z0.d, all, mul #16
+uqdecd	z0.d, pow2
+uqdecd	z0.d, pow2, mul #16
+uqdech	w0
+uqdech	w0, all, mul #16
+uqdech	w0, pow2
+uqdech	w0, pow2, mul #16
+uqdech	x0
+uqdech	x0, #14
+uqdech	x0, all, mul #16
+uqdech	x0, pow2
+uqdech	x0, vl1
+uqdech	z0.h
+uqdech	z0.h, all, mul #16
+uqdech	z0.h, pow2
+uqdech	z0.h, pow2, mul #16
+uqdecp	wzr, p15.b
+uqdecp	wzr, p15.d
+uqdecp	wzr, p15.h
+uqdecp	wzr, p15.s
+uqdecp	x0, p0.b
+uqdecp	x0, p0.d
+uqdecp	x0, p0.h
+uqdecp	x0, p0.s
+uqdecp	z0.d, p0.d
+uqdecp	z0.h, p0.h
+uqdecp	z0.s, p0.s
+uqdecw	w0
+uqdecw	w0, all, mul #16
+uqdecw	w0, pow2
+uqdecw	w0, pow2, mul #16
+uqdecw	x0
+uqdecw	x0, #14
+uqdecw	x0, all, mul #16
+uqdecw	x0, pow2
+uqdecw	x0, vl1
+uqdecw	z0.s
+uqdecw	z0.s, all, mul #16
+uqdecw	z0.s, pow2
+uqdecw	z0.s, pow2, mul #16
+uqincb	w0
+uqincb	w0, all, mul #16
+uqincb	w0, pow2
+uqincb	w0, pow2, mul #16
+uqincb	x0
+uqincb	x0, #14
+uqincb	x0, all, mul #16
+uqincb	x0, pow2
+uqincb	x0, vl1
+uqincd	w0
+uqincd	w0, all, mul #16
+uqincd	w0, pow2
+uqincd	w0, pow2, mul #16
+uqincd	x0
+uqincd	x0, #14
+uqincd	x0, all, mul #16
+uqincd	x0, pow2
+uqincd	x0, vl1
+uqincd	z0.d
+uqincd	z0.d, all, mul #16
+uqincd	z0.d, pow2
+uqincd	z0.d, pow2, mul #16
+uqinch	w0
+uqinch	w0, all, mul #16
+uqinch	w0, pow2
+uqinch	w0, pow2, mul #16
+uqinch	x0
+uqinch	x0, #14
+uqinch	x0, all, mul #16
+uqinch	x0, pow2
+uqinch	x0, vl1
+uqinch	z0.h
+uqinch	z0.h, all, mul #16
+uqinch	z0.h, pow2
+uqinch	z0.h, pow2, mul #16
+uqincp	wzr, p15.b
+uqincp	wzr, p15.d
+uqincp	wzr, p15.h
+uqincp	wzr, p15.s
+uqincp	x0, p0.b
+uqincp	x0, p0.d
+uqincp	x0, p0.h
+uqincp	x0, p0.s
+uqincp	z0.d, p0.d
+uqincp	z0.h, p0.h
+uqincp	z0.s, p0.s
+uqincw	w0
+uqincw	w0, all, mul #16
+uqincw	w0, pow2
+uqincw	w0, pow2, mul #16
+uqincw	x0
+uqincw	x0, #14
+uqincw	x0, all, mul #16
+uqincw	x0, pow2
+uqincw	x0, vl1
+uqincw	z0.s
+uqincw	z0.s, all, mul #16
+uqincw	z0.s, pow2
+uqincw	z0.s, pow2, mul #16
+uqrshl	z0.b, p0/m, z0.b, z1.b
+uqrshl	z0.h, p0/m, z0.h, z1.h
+uqrshl	z29.s, p7/m, z29.s, z30.s
+uqrshl	z31.d, p7/m, z31.d, z30.d
+uqrshlr	z0.b, p0/m, z0.b, z1.b
+uqrshlr	z0.h, p0/m, z0.h, z1.h
+uqrshlr	z29.s, p7/m, z29.s, z30.s
+uqrshlr	z31.d, p7/m, z31.d, z30.d
+uqrshrnb	z0.b, z0.h, #1
+uqrshrnb	z0.h, z0.s, #1
+uqrshrnb	z0.s, z0.d, #1
+uqrshrnb	z31.b, z31.h, #8
+uqrshrnb	z31.h, z31.s, #16
+uqrshrnb	z31.s, z31.d, #32
+uqrshrnt	z0.b, z0.h, #1
+uqrshrnt	z0.h, z0.s, #1
+uqrshrnt	z0.s, z0.d, #1
+uqrshrnt	z31.b, z31.h, #8
+uqrshrnt	z31.h, z31.s, #16
+uqrshrnt	z31.s, z31.d, #32
+uqshl	z0.b, p0/m, z0.b, #0
+uqshl	z0.b, p0/m, z0.b, z1.b
+uqshl	z0.d, p0/m, z0.d, #0
+uqshl	z0.h, p0/m, z0.h, #0
+uqshl	z0.h, p0/m, z0.h, z1.h
+uqshl	z0.s, p0/m, z0.s, #0
+uqshl	z29.s, p7/m, z29.s, z30.s
+uqshl	z31.b, p0/m, z31.b, #7
+uqshl	z31.d, p0/m, z31.d, #63
+uqshl	z31.d, p7/m, z31.d, z30.d
+uqshl	z31.h, p0/m, z31.h, #15
+uqshl	z31.s, p0/m, z31.s, #31
+uqshlr	z0.b, p0/m, z0.b, z1.b
+uqshlr	z0.h, p0/m, z0.h, z1.h
+uqshlr	z29.s, p7/m, z29.s, z30.s
+uqshlr	z31.d, p7/m, z31.d, z30.d
+uqshrnb	z0.b, z0.h, #1
+uqshrnb	z0.h, z0.s, #1
+uqshrnb	z0.s, z0.d, #1
+uqshrnb	z31.b, z31.h, #8
+uqshrnb	z31.h, z31.s, #16
+uqshrnb	z31.s, z31.d, #32
+uqshrnt	z0.b, z0.h, #1
+uqshrnt	z0.h, z0.s, #1
+uqshrnt	z0.s, z0.d, #1
+uqshrnt	z31.b, z31.h, #8
+uqshrnt	z31.h, z31.s, #16
+uqshrnt	z31.s, z31.d, #32
+uqsub	z0.b, p0/m, z0.b, z1.b
+uqsub	z0.b, z0.b, #0
+uqsub	z0.b, z0.b, z0.b
+uqsub	z0.d, z0.d, #0
+uqsub	z0.d, z0.d, #0, lsl #8
+uqsub	z0.d, z0.d, z0.d
+uqsub	z0.h, p0/m, z0.h, z1.h
+uqsub	z0.h, z0.h, #0
+uqsub	z0.h, z0.h, #0, lsl #8
+uqsub	z0.h, z0.h, z0.h
+uqsub	z0.s, z0.s, #0
+uqsub	z0.s, z0.s, #0, lsl #8
+uqsub	z0.s, z0.s, z0.s
+uqsub	z29.s, p7/m, z29.s, z30.s
+uqsub	z31.b, z31.b, #255
+uqsub	z31.d, p7/m, z31.d, z30.d
+uqsub	z31.d, z31.d, #65280
+uqsub	z31.h, z31.h, #65280
+uqsub	z31.s, z31.s, #65280
+uqsubr	z0.b, p0/m, z0.b, z1.b
+uqsubr	z0.h, p0/m, z0.h, z1.h
+uqsubr	z29.s, p7/m, z29.s, z30.s
+uqsubr	z31.d, p7/m, z31.d, z30.d
+uqxtnb	z0.b, z31.h
+uqxtnb	z0.h, z31.s
+uqxtnb	z0.s, z31.d
+uqxtnt	z0.b, z31.h
+uqxtnt	z0.h, z31.s
+uqxtnt	z0.s, z31.d
+urecpe	z31.s, p7/m, z31.s
+urhadd	z0.b, p0/m, z0.b, z1.b
+urhadd	z0.h, p0/m, z0.h, z1.h
+urhadd	z29.s, p7/m, z29.s, z30.s
+urhadd	z31.d, p7/m, z31.d, z30.d
+urshl	z0.b, p0/m, z0.b, z1.b
+urshl	z0.h, p0/m, z0.h, z1.h
+urshl	z29.s, p7/m, z29.s, z30.s
+urshl	z31.d, p7/m, z31.d, z30.d
+urshlr	z0.b, p0/m, z0.b, z1.b
+urshlr	z0.h, p0/m, z0.h, z1.h
+urshlr	z29.s, p7/m, z29.s, z30.s
+urshlr	z31.d, p7/m, z31.d, z30.d
+urshr	z0.b, p0/m, z0.b, #1
+urshr	z0.d, p0/m, z0.d, #1
+urshr	z0.h, p0/m, z0.h, #1
+urshr	z0.s, p0/m, z0.s, #1
+urshr	z31.b, p0/m, z31.b, #8
+urshr	z31.d, p0/m, z31.d, #64
+urshr	z31.h, p0/m, z31.h, #16
+urshr	z31.s, p0/m, z31.s, #32
+ursqrte	z31.s, p7/m, z31.s
+ursra	z0.b, z0.b, #1
+ursra	z0.d, z0.d, #1
+ursra	z0.h, z0.h, #1
+ursra	z0.s, z0.s, #1
+ursra	z31.b, z31.b, #8
+ursra	z31.d, z31.d, #64
+ursra	z31.h, z31.h, #16
+ursra	z31.s, z31.s, #32
+ushllb	z0.d, z0.s, #0
+ushllb	z0.h, z0.b, #0
+ushllb	z0.s, z0.h, #0
+ushllb	z31.d, z31.s, #31
+ushllb	z31.h, z31.b, #7
+ushllb	z31.s, z31.h, #15
+ushllt	z0.d, z0.s, #0
+ushllt	z0.h, z0.b, #0
+ushllt	z0.s, z0.h, #0
+ushllt	z31.d, z31.s, #31
+ushllt	z31.h, z31.b, #7
+ushllt	z31.s, z31.h, #15
+usmmla	z0.s, z1.b, z2.b
+usqadd	z0.b, p0/m, z0.b, z1.b
+usqadd	z0.h, p0/m, z0.h, z1.h
+usqadd	z29.s, p7/m, z29.s, z30.s
+usqadd	z31.d, p7/m, z31.d, z30.d
+usra	z0.b, z0.b, #1
+usra	z0.d, z0.d, #1
+usra	z0.h, z0.h, #1
+usra	z0.s, z0.s, #1
+usra	z31.b, z31.b, #8
+usra	z31.d, z31.d, #64
+usra	z31.h, z31.h, #16
+usra	z31.s, z31.s, #32
+usublb	z0.h, z1.b, z2.b
+usublb	z29.s, z30.h, z31.h
+usublb	z31.d, z31.s, z31.s
+usublt	z0.h, z1.b, z2.b
+usublt	z29.s, z30.h, z31.h
+usublt	z31.d, z31.s, z31.s
+usubwb	z0.h, z1.h, z2.b
+usubwb	z29.s, z30.s, z31.h
+usubwb	z31.d, z31.d, z31.s
+usubwt	z0.h, z1.h, z2.b
+usubwt	z29.s, z30.s, z31.h
+usubwt	z31.d, z31.d, z31.s
+uunpkhi	z31.d, z31.s
+uunpkhi	z31.h, z31.b
+uunpkhi	z31.s, z31.h
+uunpklo	z31.d, z31.s
+uunpklo	z31.h, z31.b
+uunpklo	z31.s, z31.h
+uxtb	z0.d, p0/m, z0.d
+uxtb	z0.h, p0/m, z0.h
+uxtb	z0.s, p0/m, z0.s
+uxtb	z31.d, p7/m, z31.d
+uxtb	z31.h, p7/m, z31.h
+uxtb	z31.s, p7/m, z31.s
+uxth	z0.d, p0/m, z0.d
+uxth	z0.s, p0/m, z0.s
+uxth	z31.d, p7/m, z31.d
+uxth	z31.s, p7/m, z31.s
+uxtw	z0.d, p0/m, z0.d
+uxtw	z31.d, p7/m, z31.d
+uzp1	p15.b, p15.b, p15.b
+uzp1	p15.d, p15.d, p15.d
+uzp1	p15.h, p15.h, p15.h
+uzp1	p15.s, p15.s, p15.s
+uzp1	z31.b, z31.b, z31.b
+uzp1	z31.d, z31.d, z31.d
+uzp1	z31.h, z31.h, z31.h
+uzp1	z31.s, z31.s, z31.s
+uzp2	p15.b, p15.b, p15.b
+uzp2	p15.d, p15.d, p15.d
+uzp2	p15.h, p15.h, p15.h
+uzp2	p15.s, p15.s, p15.s
+uzp2	z31.b, z31.b, z31.b
+uzp2	z31.d, z31.d, z31.d
+uzp2	z31.h, z31.h, z31.h
+uzp2	z31.s, z31.s, z31.s
+whilege	p15.b, w0, wzr
+whilege	p15.b, wzr, w0
+whilege	p15.b, x0, xzr
+whilege	p15.b, xzr, x0
+whilege	p15.d, w0, wzr
+whilege	p15.d, x0, xzr
+whilege	p15.h, w0, wzr
+whilege	p15.h, x0, xzr
+whilege	p15.s, w0, wzr
+whilege	p15.s, x0, xzr
+whilerw	p15.b, x30, x30
+whilerw	p15.d, x30, x30
+whilerw	p15.h, x30, x30
+whilerw	p15.s, x30, x30
+whilewr	p15.b, x30, x30
+whilewr	p15.d, x30, x30
+whilewr	p15.h, x30, x30
+whilewr	p15.s, x30, x30
+wrffr	p0.b
+wrffr	p15.b
+xar	z0.b, z0.b, z1.b, #1
+xar	z0.d, z0.d, z1.d, #1
+xar	z0.h, z0.h, z1.h, #1
+xar	z0.s, z0.s, z1.s, #1
+xar	z31.b, z31.b, z30.b, #8
+xar	z31.d, z31.d, z30.d, #64
+xar	z31.h, z31.h, z30.h, #16
+xar	z31.s, z31.s, z30.s, #32
+zip1	p0.b, p0.b, p0.b
+zip1	p0.d, p0.d, p0.d
+zip1	p0.h, p0.h, p0.h
+zip1	p0.s, p0.s, p0.s
+zip1	p15.b, p15.b, p15.b
+zip1	p15.d, p15.d, p15.d
+zip1	p15.h, p15.h, p15.h
+zip1	p15.s, p15.s, p15.s
+zip1	z0.b, z0.b, z0.b
+zip1	z0.d, z0.d, z0.d
+zip1	z0.h, z0.h, z0.h
+zip1	z0.s, z0.s, z0.s
+zip1	z31.b, z31.b, z31.b
+zip1	z31.d, z31.d, z31.d
+zip1	z31.h, z31.h, z31.h
+zip1	z31.s, z31.s, z31.s
+zip2	p0.b, p0.b, p0.b
+zip2	p0.d, p0.d, p0.d
+zip2	p0.h, p0.h, p0.h
+zip2	p0.s, p0.s, p0.s
+zip2	p15.b, p15.b, p15.b
+zip2	p15.d, p15.d, p15.d
+zip2	p15.h, p15.h, p15.h
+zip2	p15.s, p15.s, p15.s
+zip2	z0.b, z0.b, z0.b
+zip2	z0.d, z0.d, z0.d
+zip2	z0.h, z0.h, z0.h
+zip2	z0.s, z0.s, z0.s
+zip2	z31.b, z31.b, z31.b
+zip2	z31.d, z31.d, z31.d
+zip2	z31.h, z31.h, z31.h
+zip2	z31.s, z31.s, z31.s
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     1.00                        abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  1      3     1.00                        abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     1.00                        abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        adclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        adclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        adclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        adclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        add	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        add	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        add	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        add	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  1      3     1.00                        add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  1      3     1.00                        add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  1      3     1.00                        add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  1      3     1.00                        add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  1      3     1.00                        add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  1      3     1.00                        add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  1      3     1.00                        add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  1      3     1.00                        add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  1      3     1.00                        add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  1      3     1.00                        add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  1      3     1.00                        add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  1      3     1.00                        add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  1      3     1.00                        add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  1      3     1.00                        add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  1      3     1.00                        add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  1      3     1.00                        add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        add	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     1.00                        add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        add	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     1.00                        add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        add	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     1.00                        add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        add	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      3     1.00                        add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        addhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        addhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        addhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        addhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        addhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        addp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        addp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        addp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        addp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      1     1.00                        addpl	sp, sp, #31
+# CHECK-NEXT:  1      1     1.00                        addpl	x0, x0, #-32
+# CHECK-NEXT:  1      1     1.00                        addpl	x21, x21, #0
+# CHECK-NEXT:  1      1     1.00                        addpl	x23, x8, #-1
+# CHECK-NEXT:  1      1     1.00                        addvl	sp, sp, #31
+# CHECK-NEXT:  1      1     1.00                        addvl	x0, x0, #-32
+# CHECK-NEXT:  1      1     1.00                        addvl	x21, x21, #0
+# CHECK-NEXT:  1      1     1.00                        addvl	x23, x8, #-1
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  1      3     1.00                        adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  1      3     1.00                        aesd	z0.b, z0.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        aese	z0.b, z0.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        aesimc	z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        aesimc	z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        aesmc	z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        aesmc	z31.b, z31.b
+# CHECK-NEXT:  1      2     1.00                        and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     1.00                        and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     1.00                        and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     1.00                        and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     1.00                        and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     1.00                        and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     1.00                        and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     1.00                        and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      2     1.00                        ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     1.00                        andv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        andv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        andv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        andv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     1.00                        asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     1.00                        asr	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     1.00                        asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      3     1.00                        asr	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     1.00                        asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      3     1.00                        asr	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     1.00                        asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      3     1.00                        asr	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     1.00                        asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        asr	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     1.00                        asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     1.00                        asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     1.00                        asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     1.00                        asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     1.00                        asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        bcax	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  1      13    12.00                       bdep	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      68    67.00                       bdep	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      21    20.00                       bdep	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      37    36.00                       bdep	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      13    12.00                       bext	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      68    67.00                       bext	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      21    20.00                       bext	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      37    36.00                       bext	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      4     1.00                        bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  2      11    1.00                        bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  2      11    1.00                        bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  2      11    1.00                        bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  1      4     1.00                        bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  1      4     1.00                        bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  1      4     1.00                        bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  1      4     1.00                        bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  1      4     1.00                        bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  1      4     1.00                        bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  1      4     1.00                        bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  2      16    1.00                        bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      13    12.00                       bgrp	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      68    67.00                       bgrp	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      21    20.00                       bgrp	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      37    36.00                       bgrp	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  1      2     1.00                        brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      2     1.00                        brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      2     1.00                        brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      2     1.00                        brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      2     1.00                        brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        bsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     1.00                        bsl1n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     1.00                        bsl2n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     1.00                        cadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  1      3     1.00                        cadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  1      3     1.00                        cadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  1      3     1.00                        cadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  1      3     1.00                        cadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  1      3     1.00                        cadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  1      3     1.00                        cadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  1      3     1.00                        cadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.d, z1.h, z15.h[1], #0
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.d, z1.h, z31.h, #0
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.d, z1.h, z31.h, #180
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.d, z1.h, z31.h, #270
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.d, z1.h, z31.h, #90
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.s, z1.b, z31.b, #0
+# CHECK-NEXT:  1      4     1.00                        cdot	z0.s, z1.b, z7.b[3], #0
+# CHECK-NEXT:  1      4     1.00                        cdot	z29.d, z30.h, z0.h[0], #180
+# CHECK-NEXT:  1      4     1.00                        cdot	z31.d, z30.h, z7.h[1], #270
+# CHECK-NEXT:  1      4     1.00                        cdot	z5.d, z6.h, z3.h[0], #90
+# CHECK-NEXT:  1      4     1.00                        clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  1      4     1.00                        clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      4     1.00                        clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      4     1.00                        clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      8     2.00                        clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  1      8     2.00                        clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  1      8     2.00                        clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  1      8     2.00                        clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  1      4     1.00                        clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  1      4     1.00                        clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      4     1.00                        clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      4     1.00                        clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      8     2.00                        clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  1      8     2.00                        clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  1      8     2.00                        clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  1      8     2.00                        clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  1      4     1.00                        clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  1      4     1.00                        cmla	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  1      4     1.00                        cmla	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  1      4     1.00                        cmla	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  1      4     1.00                        cmla	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  1      4     1.00                        cmla	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  1      4     1.00                        cmla	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  1      4     1.00                        cmla	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  1      4     1.00                        cmla	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  1      4     1.00                        cmla	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      5     1.00                        cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      5     1.00                        cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      5     1.00                        cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      12    1.00                        cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      8     1.00                        cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        cntb	x0
+# CHECK-NEXT:  1      3     1.00                        cntb	x0, #28
+# CHECK-NEXT:  1      3     1.00                        cntb	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        cntb	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        cntd	x0
+# CHECK-NEXT:  1      3     1.00                        cntd	x0, #28
+# CHECK-NEXT:  1      3     1.00                        cntd	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        cntd	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        cnth	x0
+# CHECK-NEXT:  1      3     1.00                        cnth	x0, #28
+# CHECK-NEXT:  1      3     1.00                        cnth	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        cnth	x0, pow2
+# CHECK-NEXT:  1      1     1.00                        cntp	x0, p15, p0.b
+# CHECK-NEXT:  1      1     1.00                        cntp	x0, p15, p0.d
+# CHECK-NEXT:  1      1     1.00                        cntp	x0, p15, p0.h
+# CHECK-NEXT:  1      1     1.00                        cntp	x0, p15, p0.s
+# CHECK-NEXT:  1      3     1.00                        cntw	x0
+# CHECK-NEXT:  1      3     1.00                        cntw	x0, #28
+# CHECK-NEXT:  1      3     1.00                        cntw	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        cntw	x0, pow2
+# CHECK-NEXT:  1      4     1.00                        compact	z31.d, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        compact	z31.s, p7, z31.s
+# CHECK-NEXT:  1      1     1.00                        ctermeq	w30, wzr
+# CHECK-NEXT:  1      1     1.00                        ctermeq	wzr, w30
+# CHECK-NEXT:  1      1     1.00                        ctermeq	x30, xzr
+# CHECK-NEXT:  1      1     1.00                        ctermeq	xzr, x30
+# CHECK-NEXT:  1      1     1.00                        ctermne	w30, wzr
+# CHECK-NEXT:  1      1     1.00                        ctermne	wzr, w30
+# CHECK-NEXT:  1      1     1.00                        ctermne	x30, xzr
+# CHECK-NEXT:  1      1     1.00                        ctermne	xzr, x30
+# CHECK-NEXT:  1      3     1.00                        decb	x0
+# CHECK-NEXT:  1      3     1.00                        decb	x0, #14
+# CHECK-NEXT:  1      3     1.00                        decb	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        decb	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        decb	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        decd	x0
+# CHECK-NEXT:  1      3     1.00                        decd	x0, #14
+# CHECK-NEXT:  1      3     1.00                        decd	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        decd	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        decd	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        dech	x0
+# CHECK-NEXT:  1      3     1.00                        dech	x0, #14
+# CHECK-NEXT:  1      3     1.00                        dech	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        dech	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        dech	x0, vl1
+# CHECK-NEXT:  1      1     1.00                        decp	x0, p0.b
+# CHECK-NEXT:  1      1     1.00                        decp	x0, p0.d
+# CHECK-NEXT:  1      1     1.00                        decp	x0, p0.h
+# CHECK-NEXT:  1      1     1.00                        decp	x0, p0.s
+# CHECK-NEXT:  1      1     1.00                        decp	xzr, p15.b
+# CHECK-NEXT:  1      1     1.00                        decp	xzr, p15.d
+# CHECK-NEXT:  1      1     1.00                        decp	xzr, p15.h
+# CHECK-NEXT:  1      1     1.00                        decp	xzr, p15.s
+# CHECK-NEXT:  1      3     1.00                        decp	z31.d, p15.d
+# CHECK-NEXT:  1      3     1.00                        decp	z31.h, p15.h
+# CHECK-NEXT:  1      3     1.00                        decp	z31.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        decw	x0
+# CHECK-NEXT:  1      3     1.00                        decw	x0, #14
+# CHECK-NEXT:  1      3     1.00                        decw	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        decw	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        decw	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      4     1.00                        dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  1      4     1.00                        dupm	z23.h, #0xfff9
+# CHECK-NEXT:  1      4     1.00                        dupm	z5.b, #0xf9
+# CHECK-NEXT:  1      2     1.00                        eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     1.00                        eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     1.00                        eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     1.00                        eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     1.00                        eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     1.00                        eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     1.00                        eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     1.00                        eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      4     1.00                        eor3	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        eorbt	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        eorbt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        eorbt	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        eorbt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     1.00                        eortb	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        eortb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        eortb	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        eortb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        eorv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        eorv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        eorv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        eorv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     1.00                        ext	z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  1      3     1.00                        ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  1      3     1.00                        ext	z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  1      4     1.00                        fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      5     1.00                        facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      8     5.00                        fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      32    29.00                       fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      16    13.00                       fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      4     1.00                        faddp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        faddp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        faddp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        faddv	d0, p7, z31.d
+# CHECK-NEXT:  1      12    11.00                       faddv	h0, p7, z31.h
+# CHECK-NEXT:  1      8     5.00                        faddv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  1      4     1.00                        fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  1      4     1.00                        fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  1      4     1.00                        fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     1.00                        fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     1.00                        fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  1      4     1.00                        fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  1      4     1.00                        fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  1      4     1.00                        fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  1      4     1.00                        fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  1      4     1.00                        fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  1      4     1.00                        fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     1.00                        fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     1.00                        fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  1      4     1.00                        fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  1      5     1.00                        fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      5     1.00                        fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      5     1.00                        fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      5     1.00                        fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      5     1.00                        fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtlt	z0.s, p0/m, z1.h
+# CHECK-NEXT:  1      4     1.00                        fcvtlt	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        fcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      4     1.00                        fcvtnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        fcvtx	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvtx	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        fcvtxnt	z0.s, p0/m, z1.d
+# CHECK-NEXT:  1      4     1.00                        fcvtxnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      22    19.00                       fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      8     5.00                        fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      13    10.00                       fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      22    19.00                       fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      8     5.00                        fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      13    10.00                       fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fexpa	z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fexpa	z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fexpa	z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        flogb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        flogb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        flogb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        fmaxnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fmaxp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        fmaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        fminnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fminnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        fminnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        fminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        fminp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        fminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmlalb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     1.00                        fmlalb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmlalb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmlalt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     1.00                        fmlalt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmlalt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmlslb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     1.00                        fmlslb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmlslb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmlslt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     1.00                        fmlslt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmlslt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, #0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  1      4     1.00                        fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  1      4     1.00                        fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  1      4     1.00                        fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        frecpe	z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        frecpe	z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        frecpe	z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        frsqrte	z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        frsqrte	z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        frsqrte	z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      25    19.00                       fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      11    5.00                        fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      14    9.00                        fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     1.00                        fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     1.00                        fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     1.00                        fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     1.00                        ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  1      4     1.00                        ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  1      4     1.00                        ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  1      4     1.00                        ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      8     2.00                        histcnt	z0.s, p0/z, z1.s, z2.s
+# CHECK-NEXT:  1      8     2.00                        histcnt	z29.d, p7/z, z30.d, z31.d
+# CHECK-NEXT:  1      8     2.00                        histseg	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        incb	x0
+# CHECK-NEXT:  1      3     1.00                        incb	x0, #14
+# CHECK-NEXT:  1      3     1.00                        incb	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        incb	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        incb	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        incd	x0
+# CHECK-NEXT:  1      3     1.00                        incd	x0, #14
+# CHECK-NEXT:  1      3     1.00                        incd	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        incd	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        incd	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        incd	z0.d
+# CHECK-NEXT:  1      3     1.00                        incd	z0.d, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        inch	x0
+# CHECK-NEXT:  1      3     1.00                        inch	x0, #14
+# CHECK-NEXT:  1      3     1.00                        inch	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        inch	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        inch	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        inch	z0.h
+# CHECK-NEXT:  1      3     1.00                        inch	z0.h, all, mul #16
+# CHECK-NEXT:  1      1     1.00                        incp	x0, p0.b
+# CHECK-NEXT:  1      1     1.00                        incp	x0, p0.d
+# CHECK-NEXT:  1      1     1.00                        incp	x0, p0.h
+# CHECK-NEXT:  1      1     1.00                        incp	x0, p0.s
+# CHECK-NEXT:  1      1     1.00                        incp	xzr, p15.b
+# CHECK-NEXT:  1      1     1.00                        incp	xzr, p15.d
+# CHECK-NEXT:  1      1     1.00                        incp	xzr, p15.h
+# CHECK-NEXT:  1      1     1.00                        incp	xzr, p15.s
+# CHECK-NEXT:  1      3     1.00                        incp	z31.d, p15.d
+# CHECK-NEXT:  1      3     1.00                        incp	z31.h, p15.h
+# CHECK-NEXT:  1      3     1.00                        incp	z31.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        incw	x0
+# CHECK-NEXT:  1      3     1.00                        incw	x0, #14
+# CHECK-NEXT:  1      3     1.00                        incw	x0, all, mul #16
+# CHECK-NEXT:  1      3     1.00                        incw	x0, pow2
+# CHECK-NEXT:  1      3     1.00                        incw	x0, vl1
+# CHECK-NEXT:  1      3     1.00                        incw	z0.s
+# CHECK-NEXT:  1      3     1.00                        incw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        index	z0.b, #0, #0
+# CHECK-NEXT:  1      4     1.00                        index	z0.d, #0, #0
+# CHECK-NEXT:  1      4     1.00                        index	z0.h, #0, #0
+# CHECK-NEXT:  1      4     1.00                        index	z0.h, w0, w0
+# CHECK-NEXT:  1      4     1.00                        index	z0.s, #0, #0
+# CHECK-NEXT:  1      4     1.00                        index	z21.b, w10, w21
+# CHECK-NEXT:  1      4     1.00                        index	z21.d, x10, x21
+# CHECK-NEXT:  1      4     1.00                        index	z21.s, w10, w21
+# CHECK-NEXT:  1      4     1.00                        index	z23.b, #13, w8
+# CHECK-NEXT:  1      4     1.00                        index	z23.b, w13, #8
+# CHECK-NEXT:  1      4     1.00                        index	z23.d, #13, x8
+# CHECK-NEXT:  1      4     1.00                        index	z23.d, x13, #8
+# CHECK-NEXT:  1      4     1.00                        index	z23.h, #13, w8
+# CHECK-NEXT:  1      4     1.00                        index	z23.h, w13, #8
+# CHECK-NEXT:  1      4     1.00                        index	z23.s, #13, w8
+# CHECK-NEXT:  1      4     1.00                        index	z23.s, w13, #8
+# CHECK-NEXT:  1      4     1.00                        index	z31.b, #-1, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.b, #-1, wzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.b, wzr, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.b, wzr, wzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.d, #-1, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.d, #-1, xzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.d, xzr, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.d, xzr, xzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.h, #-1, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.h, #-1, wzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.h, wzr, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.h, wzr, wzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.s, #-1, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.s, #-1, wzr
+# CHECK-NEXT:  1      4     1.00                        index	z31.s, wzr, #-1
+# CHECK-NEXT:  1      4     1.00                        index	z31.s, wzr, wzr
+# CHECK-NEXT:  1      8     2.00                        insr	z0.b, w0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.d, x0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.h, w0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.s, w0
+# CHECK-NEXT:  1      4     1.00                        insr	z31.b, b31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.b, wzr
+# CHECK-NEXT:  1      4     1.00                        insr	z31.d, d31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.d, xzr
+# CHECK-NEXT:  1      4     1.00                        insr	z31.h, h31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.h, wzr
+# CHECK-NEXT:  1      4     1.00                        insr	z31.s, s31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.s, wzr
+# CHECK-NEXT:  1      4     1.00                        lasta	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        lasta	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        lasta	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        lasta	s0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.b
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.h
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lasta	x0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        lastb	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        lastb	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        lastb	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        lastb	s0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.b
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.h
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lastb	x0, p7, z31.d
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     9.00    *                   ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     1.00    *                   ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *                   ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      9     9.00    *                   ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      3     1.00    *                   ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  1      9     9.00    *                   ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  1      3     2.00    *                   ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      3     2.00    *                   ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      3     2.00    *                   ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      3     2.00    *                   ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  1      3     1.00    *             U     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *             U     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p0, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	z0, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        mad	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        mad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        mad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        mad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      9     2.00                        match	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      9     2.00                        match	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      9     2.00                        match	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  1      9     2.00                        match	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        mla	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        mla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        mla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     1.00                        mla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        mla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        mla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        mla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        mls	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        mls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        mls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     1.00                        mls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        mls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        mls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        mls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      2     1.00                        mov	p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  1      2     1.00                        mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      2     1.00                        mov	p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  1      2     1.00                        mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, b0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, p0/m, b0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, p0/m, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, p0/z, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  1      4     1.00                        mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  1      4     1.00                        mov	z0.d, #32768
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, d0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, p0/m, d0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, p0/m, x0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, x0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, #-256
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, #32512
+# CHECK-NEXT:  1      4     1.00                        mov	z0.h, #32767
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, h0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, p0/m, h0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, p0/m, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.q, q0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  1      4     1.00                        mov	z0.s, #32768
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, p0/m, s0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, p0/m, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, s0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, w0
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p0/z, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p0/z, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p0/z, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, p7/m, b31
+# CHECK-NEXT:  1      3     1.00                        movprfx	z31, z6
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, z31.b[63]
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, p7/m, d31
+# CHECK-NEXT:  1      3     1.00                        movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, p7/m, sp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, sp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, z31.d[7]
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, p7/m, h31
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, z31.h[31]
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, p7/m, s31
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, wsp
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, z31.s[15]
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, #-1
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, p0/z, #127
+# CHECK-NEXT:  1      3     1.00                        mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  1      3     1.00                        mov	z5.d, #-6
+# CHECK-NEXT:  1      3     1.00                        mov	z5.h, #-6
+# CHECK-NEXT:  1      3     1.00                        mov	z5.q, z17.q[3]
+# CHECK-NEXT:  1      3     1.00                        mov	z5.s, #-6
+# CHECK-NEXT:  1      2     1.00                        movs	p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      2     1.00                        movs	p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  1      4     1.00                        msb	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        msb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        msb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        msb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL1, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL12, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL2, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL3, x3
+# CHECK-NEXT:  1      4     1.00                        mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        mul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        mul	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        mul	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        mul	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        mul	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        mul	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  1      4     1.00                        mul	z31.b, z31.b, #127
+# CHECK-NEXT:  1      4     1.00                        mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  1      4     1.00                        mul	z31.d, z31.d, #127
+# CHECK-NEXT:  1      4     1.00                        mul	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  1      4     1.00                        mul	z31.h, z31.h, #127
+# CHECK-NEXT:  1      4     1.00                        mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  1      4     1.00                        mul	z31.s, z31.s, #127
+# CHECK-NEXT:  1      2     1.00                        nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     1.00                        nbsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     1.00                        neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  1      3     1.00                        neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     1.00                        neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      9     2.00                        nmatch	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      9     2.00                        nmatch	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      9     2.00                        nmatch	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  1      9     2.00                        nmatch	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  1      2     1.00                        nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      2     1.00                        not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     1.00                        not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      2     1.00                        nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      2     1.00                        nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      2     1.00                        orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     1.00                        orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     1.00                        orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     1.00                        orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     1.00                        orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     1.00                        orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     1.00                        orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     1.00                        orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     1.00                        orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      2     1.00                        orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     1.00                        orv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        orv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        orv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        orv	s0, p7, z31.s
+# CHECK-NEXT:  1      2     1.00                        pfalse	p15.b
+# CHECK-NEXT:  1      2     1.00                        pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  1      2     1.00                        pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  1      4     1.00                        pmul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        pmul	z29.b, z30.b, z31.b
+# CHECK-NEXT:  1      9     1.00                        pmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      9     1.00                        pmullb	z29.q, z30.d, z31.d
+# CHECK-NEXT:  1      9     1.00                        pmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      9     1.00                        pmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      9     1.00                        pmullt	z29.q, z30.d, z31.d
+# CHECK-NEXT:  1      9     1.00                        pmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  1      2     1.00                        pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  1      2     1.00                        pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  1      2     1.00                        pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  1      2     1.00                        pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#14, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#15, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#6, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#7, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#7, p3, [z13.s, #31]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	#7, p3, [z13.s]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1keep, p0, [x0, z0.d]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3strm, p5, [z10.d, #31]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pldl3strm, p5, [z10.d]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfb	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#14, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#15, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#15, p7, [z31.d, #248]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#15, p7, [z31.s, #248]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#6, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	#7, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfd	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#14, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#15, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#15, p7, [z31.d, #62]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#15, p7, [z31.s, #62]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#6, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	#7, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfh	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#14, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#15, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#15, p7, [z31.d, #124]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#15, p7, [z31.s, #124]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#6, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#7, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	#7, p3, [x13, z8.d, uxtw #2]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     1.00    *      *      U     prfw	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      2     1.00                        ptest	p15, p0.b
+# CHECK-NEXT:  1      2     1.00                        ptest	p15, p15.b
+# CHECK-NEXT:  1      2     1.00                        ptrue	p0.b, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrue	p0.d, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrue	p0.h, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrue	p0.s, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrue	p15.b
+# CHECK-NEXT:  1      2     1.00                        ptrue	p15.d
+# CHECK-NEXT:  1      2     1.00                        ptrue	p15.h
+# CHECK-NEXT:  1      2     1.00                        ptrue	p15.s
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #14
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #15
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #16
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #17
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #18
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #19
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #20
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #21
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #22
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #23
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #24
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #25
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #26
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #27
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, #28
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, mul3
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, mul4
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl1
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl128
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl16
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl2
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl256
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl3
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl32
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl4
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl5
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl6
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl64
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl7
+# CHECK-NEXT:  1      2     1.00                        ptrue	p7.s, vl8
+# CHECK-NEXT:  1      2     1.00                        ptrues	p0.b, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrues	p0.d, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrues	p0.h, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrues	p0.s, pow2
+# CHECK-NEXT:  1      2     1.00                        ptrues	p15.b
+# CHECK-NEXT:  1      2     1.00                        ptrues	p15.d
+# CHECK-NEXT:  1      2     1.00                        ptrues	p15.h
+# CHECK-NEXT:  1      2     1.00                        ptrues	p15.s
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #14
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #15
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #16
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #17
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #18
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #19
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #20
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #21
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #22
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #23
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #24
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #25
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #26
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #27
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, #28
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, mul3
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, mul4
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl1
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl128
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl16
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl2
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl256
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl3
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl32
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl4
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl5
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl6
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl64
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl7
+# CHECK-NEXT:  1      2     1.00                        ptrues	p7.s, vl8
+# CHECK-NEXT:  1      2     1.00                        punpkhi	p0.h, p0.b
+# CHECK-NEXT:  1      2     1.00                        punpkhi	p15.h, p15.b
+# CHECK-NEXT:  1      2     1.00                        punpklo	p0.h, p0.b
+# CHECK-NEXT:  1      2     1.00                        punpklo	p15.h, p15.b
+# CHECK-NEXT:  1      8     1.00                        raddhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        raddhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        raddhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        raddhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        raddhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        raddhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      9     1.00                        rax1	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     1.00                        rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      1     1.00    *             U     rdffr	p0.b
+# CHECK-NEXT:  1      3     1.00    *             U     rdffr	p0.b, p0/z
+# CHECK-NEXT:  1      1     1.00    *             U     rdffr	p15.b
+# CHECK-NEXT:  1      3     1.00    *             U     rdffr	p15.b, p15/z
+# CHECK-NEXT:  1      3     1.00                  U     rdffrs	p0.b, p0/z
+# CHECK-NEXT:  1      3     1.00                  U     rdffrs	p15.b, p15/z
+# CHECK-NEXT:  1      1     1.00                        rdvl	x0, #0
+# CHECK-NEXT:  1      1     1.00                        rdvl	x21, #-32
+# CHECK-NEXT:  1      1     1.00                        rdvl	x23, #31
+# CHECK-NEXT:  1      1     1.00                        rdvl	xzr, #-1
+# CHECK-NEXT:  1      3     1.00                        rev	z0.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        rev	z0.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        rev	z0.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        rev	z0.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        rshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        rshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      8     1.00                        rsubhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        rsubhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        rsubhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        rsubhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        rsubhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        rsubhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      6     2.00                        saba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        saba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      6     2.00                        saba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      6     2.00                        saba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        sabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        sabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        sabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      6     2.00                        sabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        sabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        sabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        sabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     1.00                        sabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        sabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     1.00                        sabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      7     2.00                        sadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  1      7     2.00                        sadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  1      7     2.00                        sadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        saddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddlbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddlbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        saddlbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        saddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        saddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        saddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        sbclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sbclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sbclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sbclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      26    23.00                       sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      26    23.00                       sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  1      3     1.00                        sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  1      3     1.00                        sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  1      3     1.00                        sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT:  1      1     1.00           *      U     setffr
+# CHECK-NEXT:  1      3     1.00                        shadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        shadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        shadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        shadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        shrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        shrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        shrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        shrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      3     1.00                        shrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      3     1.00                        shrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      3     1.00                        shrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        shrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        shrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        shrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      3     1.00                        shrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      3     1.00                        shrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      3     1.00                        shsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        shsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        shsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        shsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        shsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        shsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        shsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        shsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        sli	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        sli	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        sli	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        sli	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        sli	z31.b, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        sli	z31.d, z31.d, #63
+# CHECK-NEXT:  1      3     1.00                        sli	z31.h, z31.h, #15
+# CHECK-NEXT:  1      3     1.00                        sli	z31.s, z31.s, #31
+# CHECK-NEXT:  1      9     1.00                        sm4e	z0.s, z0.s, z31.s
+# CHECK-NEXT:  1      9     1.00                        sm4ekey	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  1      3     1.00                        smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  1      3     1.00                        smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  1      3     1.00                        smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  1      3     1.00                        smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        smax	z31.b, z31.b, #127
+# CHECK-NEXT:  1      3     1.00                        smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        smax	z31.d, z31.d, #127
+# CHECK-NEXT:  1      3     1.00                        smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        smax	z31.h, z31.h, #127
+# CHECK-NEXT:  1      3     1.00                        smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        smax	z31.s, z31.s, #127
+# CHECK-NEXT:  1      3     1.00                        smaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        smaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        smaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        smaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        smaxv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        smaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        smaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        smaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     1.00                        smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  1      3     1.00                        smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  1      3     1.00                        smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  1      3     1.00                        smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  1      3     1.00                        smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        smin	z31.b, z31.b, #127
+# CHECK-NEXT:  1      3     1.00                        smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        smin	z31.d, z31.d, #127
+# CHECK-NEXT:  1      3     1.00                        smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        smin	z31.h, z31.h, #127
+# CHECK-NEXT:  1      3     1.00                        smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        smin	z31.s, z31.s, #127
+# CHECK-NEXT:  1      3     1.00                        sminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        sminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        sminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        sminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sminv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        sminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        sminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        sminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        smlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        smlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        smlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        smlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        smlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        smullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        smullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        smullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        smullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        smullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        smullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        smullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        splice	z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  1      4     1.00                        splice	z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  1      4     1.00                        splice	z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  1      4     1.00                        splice	z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  1      4     1.00                        splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqabs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        sqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     1.00                        sqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     1.00                        sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     1.00                        sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     1.00                        sqcadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdecd	z0.d
+# CHECK-NEXT:  1      4     1.00                        sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdecd	z0.d, pow2
+# CHECK-NEXT:  1      4     1.00                        sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdech	z0.h
+# CHECK-NEXT:  1      4     1.00                        sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdech	z0.h, pow2
+# CHECK-NEXT:  1      4     1.00                        sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      9     1.00                        sqdecp	x0, p0.b
+# CHECK-NEXT:  1      9     1.00                        sqdecp	x0, p0.d
+# CHECK-NEXT:  1      9     1.00                        sqdecp	x0, p0.h
+# CHECK-NEXT:  1      9     1.00                        sqdecp	x0, p0.s
+# CHECK-NEXT:  1      9     1.00                        sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  1      9     1.00                        sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  1      9     1.00                        sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  1      9     1.00                        sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  1      3     1.00                        sqdecp	z0.d, p0.d
+# CHECK-NEXT:  1      3     1.00                        sqdecp	z0.h, p0.h
+# CHECK-NEXT:  1      3     1.00                        sqdecp	z0.s, p0.s
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdecw	z0.s
+# CHECK-NEXT:  1      4     1.00                        sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdecw	z0.s, pow2
+# CHECK-NEXT:  1      4     1.00                        sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqdmlalb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmlalbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlalbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlalbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlalt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmlslb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmlslbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlslbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlslbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlslt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqdmlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqdmullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        sqdmullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqdmullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        sqdmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        sqdmullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqdmullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqdmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqincd	z0.d
+# CHECK-NEXT:  1      4     1.00                        sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqincd	z0.d, pow2
+# CHECK-NEXT:  1      4     1.00                        sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqinch	z0.h
+# CHECK-NEXT:  1      4     1.00                        sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqinch	z0.h, pow2
+# CHECK-NEXT:  1      4     1.00                        sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      9     1.00                        sqincp	x0, p0.b
+# CHECK-NEXT:  1      9     1.00                        sqincp	x0, p0.d
+# CHECK-NEXT:  1      9     1.00                        sqincp	x0, p0.h
+# CHECK-NEXT:  1      9     1.00                        sqincp	x0, p0.s
+# CHECK-NEXT:  1      9     1.00                        sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  1      9     1.00                        sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  1      9     1.00                        sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  1      9     1.00                        sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  1      3     1.00                        sqincp	z0.d, p0.d
+# CHECK-NEXT:  1      3     1.00                        sqincp	z0.h, p0.h
+# CHECK-NEXT:  1      3     1.00                        sqincp	z0.s, p0.s
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, #14
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, w0
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, w0, pow2
+# CHECK-NEXT:  1      5     1.00                        sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqincw	z0.s
+# CHECK-NEXT:  1      4     1.00                        sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqincw	z0.s, pow2
+# CHECK-NEXT:  1      4     1.00                        sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        sqneg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  1      4     1.00                        sqrdcmlah	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqrdmlah	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqrdmlsh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqrdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqrshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqrshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        sqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     1.00                        sqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     1.00                        sqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     1.00                        sqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     1.00                        sqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     1.00                        sqshlu	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        sqshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        sqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     1.00                        sqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     1.00                        sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     1.00                        sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     1.00                        sqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        sqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        sqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        sqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sqxtnb	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqxtnb	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqxtnb	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqxtnt	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqxtnt	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqxtnt	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqxtunb	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqxtunb	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqxtunb	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        sqxtunt	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        sqxtunt	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        sqxtunt	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        srhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        srhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        srhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        srhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        sri	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     1.00                        sri	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     1.00                        sri	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     1.00                        sri	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     1.00                        sri	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     1.00                        sri	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     1.00                        sri	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     1.00                        sri	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     1.00                        srshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        srshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        srshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        srshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        srshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        srshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        srshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        srshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        srshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     1.00                        srshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        srshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        srshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        srshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     1.00                        srshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     1.00                        srshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     1.00                        srshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      7     2.00                        srsra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      7     2.00                        srsra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      7     2.00                        srsra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      7     2.00                        srsra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      7     2.00                        srsra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      7     2.00                        srsra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      7     2.00                        srsra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      7     2.00                        srsra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        sshllb	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        sshllb	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        sshllb	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        sshllb	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        sshllb	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        sshllb	z31.s, z31.h, #15
+# CHECK-NEXT:  1      3     1.00                        sshllt	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        sshllt	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        sshllt	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        sshllt	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        sshllt	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        sshllt	z31.s, z31.h, #15
+# CHECK-NEXT:  1      4     1.00                        ssra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      4     1.00                        ssra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        ssra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        ssra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        ssra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      4     1.00                        ssra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      4     1.00                        ssra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      4     1.00                        ssra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     1.00                        ssublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        ssublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        ssublbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        ssublbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        ssublbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        ssublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        ssubltb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        ssubltb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        ssubltb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        ssubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        ssubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        ssubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        ssubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     1.00           *            str	p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	z0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  1      3     1.00                        sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        sub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        sub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        sub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        sub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  1      3     1.00                        sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  1      3     1.00                        sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  1      3     1.00                        sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  1      3     1.00                        sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  1      3     1.00                        sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  1      3     1.00                        sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  1      3     1.00                        sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  1      3     1.00                        sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  1      3     1.00                        sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  1      3     1.00                        sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  1      3     1.00                        sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     1.00                        sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  1      3     1.00                        sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  1      3     1.00                        sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  1      3     1.00                        sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  1      3     1.00                        sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        sub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     1.00                        sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     1.00                        sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     1.00                        sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      3     1.00                        sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        subhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        subhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        subhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        subhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        subhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        subhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        subr	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        subr	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     1.00                        subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        subr	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        subr	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     1.00                        subr	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     1.00                        subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     1.00                        subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     1.00                        subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     1.00                        sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  1      4     1.00                        sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        sunpklo	z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        sunpklo	z31.h, z31.b
+# CHECK-NEXT:  1      4     1.00                        sunpklo	z31.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        suqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        suqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        suqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        suqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     1.00                        sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     1.00                        tbl	z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  1      4     1.00                        tbl	z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  1      4     1.00                        tbl	z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  1      4     1.00                        tbl	z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  1      4     1.00                        tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  1      4     1.00                        tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  1      4     1.00                        tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  1      4     1.00                        tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  1      8     1.00                        tbx	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        tbx	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        tbx	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        tbx	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        uaba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        uaba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      6     2.00                        uaba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      6     2.00                        uaba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        uabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        uabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        uabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      6     2.00                        uabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      6     2.00                        uabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      6     2.00                        uabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        uabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     1.00                        uabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        uabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     1.00                        uabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      7     2.00                        uadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  1      7     2.00                        uadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  1      7     2.00                        uadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        uaddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        uaddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        uaddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        uaddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     1.00                        ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      26    23.00                       udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      26    23.00                       udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     1.00                        udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  1      3     1.00                        uhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        uhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        uhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        uhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        uhsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        uhsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        uhsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        uhsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        uhsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        uhsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        uhsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        uhsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     1.00                        umax	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        umax	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     1.00                        umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        umaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        umaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        umaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        umaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        umaxv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        umaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        umaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        umaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     1.00                        umin	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        umin	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     1.00                        umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     1.00                        uminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     1.00                        uminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     1.00                        uminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     1.00                        uminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uminv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        uminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        uminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        uminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     1.00                        umlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        umlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        umlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        umlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        umlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        umlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        umlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        umlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     1.00                        umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     1.00                        umullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        umullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        umullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        umullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     1.00                        umullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        umullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     1.00                        umullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        umullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        uqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     1.00                        uqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     1.00                        uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     1.00                        uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      5     1.00                        uqdecb	w0
+# CHECK-NEXT:  1      5     1.00                        uqdecb	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecb	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecb	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecb	x0
+# CHECK-NEXT:  1      5     1.00                        uqdecb	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqdecb	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecb	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecb	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        uqdecd	w0
+# CHECK-NEXT:  1      5     1.00                        uqdecd	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecd	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecd	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecd	x0
+# CHECK-NEXT:  1      5     1.00                        uqdecd	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqdecd	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecd	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecd	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqdecd	z0.d
+# CHECK-NEXT:  1      4     1.00                        uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqdecd	z0.d, pow2
+# CHECK-NEXT:  1      4     1.00                        uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdech	w0
+# CHECK-NEXT:  1      5     1.00                        uqdech	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdech	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdech	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdech	x0
+# CHECK-NEXT:  1      5     1.00                        uqdech	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqdech	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdech	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdech	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqdech	z0.h
+# CHECK-NEXT:  1      4     1.00                        uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqdech	z0.h, pow2
+# CHECK-NEXT:  1      4     1.00                        uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      9     1.00                        uqdecp	wzr, p15.b
+# CHECK-NEXT:  1      9     1.00                        uqdecp	wzr, p15.d
+# CHECK-NEXT:  1      9     1.00                        uqdecp	wzr, p15.h
+# CHECK-NEXT:  1      9     1.00                        uqdecp	wzr, p15.s
+# CHECK-NEXT:  1      9     1.00                        uqdecp	x0, p0.b
+# CHECK-NEXT:  1      9     1.00                        uqdecp	x0, p0.d
+# CHECK-NEXT:  1      9     1.00                        uqdecp	x0, p0.h
+# CHECK-NEXT:  1      9     1.00                        uqdecp	x0, p0.s
+# CHECK-NEXT:  1      3     1.00                        uqdecp	z0.d, p0.d
+# CHECK-NEXT:  1      3     1.00                        uqdecp	z0.h, p0.h
+# CHECK-NEXT:  1      3     1.00                        uqdecp	z0.s, p0.s
+# CHECK-NEXT:  1      5     1.00                        uqdecw	w0
+# CHECK-NEXT:  1      5     1.00                        uqdecw	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecw	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecw	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecw	x0
+# CHECK-NEXT:  1      5     1.00                        uqdecw	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqdecw	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqdecw	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqdecw	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqdecw	z0.s
+# CHECK-NEXT:  1      4     1.00                        uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqdecw	z0.s, pow2
+# CHECK-NEXT:  1      4     1.00                        uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincb	w0
+# CHECK-NEXT:  1      5     1.00                        uqincb	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincb	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincb	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincb	x0
+# CHECK-NEXT:  1      5     1.00                        uqincb	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqincb	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincb	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincb	x0, vl1
+# CHECK-NEXT:  1      5     1.00                        uqincd	w0
+# CHECK-NEXT:  1      5     1.00                        uqincd	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincd	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincd	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincd	x0
+# CHECK-NEXT:  1      5     1.00                        uqincd	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqincd	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincd	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincd	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqincd	z0.d
+# CHECK-NEXT:  1      4     1.00                        uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqincd	z0.d, pow2
+# CHECK-NEXT:  1      4     1.00                        uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqinch	w0
+# CHECK-NEXT:  1      5     1.00                        uqinch	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqinch	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqinch	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqinch	x0
+# CHECK-NEXT:  1      5     1.00                        uqinch	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqinch	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqinch	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqinch	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqinch	z0.h
+# CHECK-NEXT:  1      4     1.00                        uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqinch	z0.h, pow2
+# CHECK-NEXT:  1      4     1.00                        uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      9     1.00                        uqincp	wzr, p15.b
+# CHECK-NEXT:  1      9     1.00                        uqincp	wzr, p15.d
+# CHECK-NEXT:  1      9     1.00                        uqincp	wzr, p15.h
+# CHECK-NEXT:  1      9     1.00                        uqincp	wzr, p15.s
+# CHECK-NEXT:  1      9     1.00                        uqincp	x0, p0.b
+# CHECK-NEXT:  1      9     1.00                        uqincp	x0, p0.d
+# CHECK-NEXT:  1      9     1.00                        uqincp	x0, p0.h
+# CHECK-NEXT:  1      9     1.00                        uqincp	x0, p0.s
+# CHECK-NEXT:  1      3     1.00                        uqincp	z0.d, p0.d
+# CHECK-NEXT:  1      3     1.00                        uqincp	z0.h, p0.h
+# CHECK-NEXT:  1      3     1.00                        uqincp	z0.s, p0.s
+# CHECK-NEXT:  1      5     1.00                        uqincw	w0
+# CHECK-NEXT:  1      5     1.00                        uqincw	w0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincw	w0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincw	w0, pow2, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincw	x0
+# CHECK-NEXT:  1      5     1.00                        uqincw	x0, #14
+# CHECK-NEXT:  1      5     1.00                        uqincw	x0, all, mul #16
+# CHECK-NEXT:  1      5     1.00                        uqincw	x0, pow2
+# CHECK-NEXT:  1      5     1.00                        uqincw	x0, vl1
+# CHECK-NEXT:  1      4     1.00                        uqincw	z0.s
+# CHECK-NEXT:  1      4     1.00                        uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqincw	z0.s, pow2
+# CHECK-NEXT:  1      4     1.00                        uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     1.00                        uqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        uqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        uqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        uqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     1.00                        uqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     1.00                        uqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     1.00                        uqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     1.00                        uqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        uqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     1.00                        uqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     1.00                        uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     1.00                        uqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     1.00                        uqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     1.00                        uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     1.00                        uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     1.00                        uqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        uqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        uqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        uqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uqxtnb	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        uqxtnb	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        uqxtnb	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        uqxtnt	z0.b, z31.h
+# CHECK-NEXT:  1      4     1.00                        uqxtnt	z0.h, z31.s
+# CHECK-NEXT:  1      4     1.00                        uqxtnt	z0.s, z31.d
+# CHECK-NEXT:  1      4     1.00                        urecpe	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     1.00                        urhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        urhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        urhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        urhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        urshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        urshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        urshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        urshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        urshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        urshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        urshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        urshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        urshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     1.00                        urshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        urshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        urshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        urshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     1.00                        urshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     1.00                        urshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     1.00                        urshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      4     1.00                        ursqrte	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      7     2.00                        ursra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      7     2.00                        ursra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      7     2.00                        ursra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      7     2.00                        ursra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      7     2.00                        ursra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      7     2.00                        ursra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      7     2.00                        ursra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      7     2.00                        ursra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     1.00                        ushllb	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        ushllb	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        ushllb	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        ushllb	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        ushllb	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        ushllb	z31.s, z31.h, #15
+# CHECK-NEXT:  1      3     1.00                        ushllt	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     1.00                        ushllt	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     1.00                        ushllt	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     1.00                        ushllt	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     1.00                        ushllt	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     1.00                        ushllt	z31.s, z31.h, #15
+# CHECK-NEXT:  1      4     1.00                        usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        usqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     1.00                        usqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     1.00                        usqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     1.00                        usqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        usra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      4     1.00                        usra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      4     1.00                        usra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      4     1.00                        usra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      4     1.00                        usra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      4     1.00                        usra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      4     1.00                        usra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      4     1.00                        usra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     1.00                        usublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        usublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        usublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        usublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     1.00                        usublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     1.00                        usublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        usubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        usubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        usubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        usubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      4     1.00                        usubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        usubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  1      4     1.00                        uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  1      4     1.00                        uunpklo	z31.d, z31.s
+# CHECK-NEXT:  1      4     1.00                        uunpklo	z31.h, z31.b
+# CHECK-NEXT:  1      4     1.00                        uunpklo	z31.s, z31.h
+# CHECK-NEXT:  1      3     1.00                        uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     1.00                        uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     1.00                        uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     1.00                        uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     1.00                        uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     1.00                        uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     1.00                        uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      2     1.00                        uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.b, w0, wzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.b, wzr, w0
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.b, x0, xzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.b, xzr, x0
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.d, w0, wzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.d, x0, xzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.h, w0, wzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.h, x0, xzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.s, w0, wzr
+# CHECK-NEXT:  1      2     1.00                        whilege	p15.s, x0, xzr
+# CHECK-NEXT:  1      2     1.00                        whilerw	p15.b, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilerw	p15.d, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilerw	p15.h, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilerw	p15.s, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilewr	p15.b, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilewr	p15.d, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilewr	p15.h, x30, x30
+# CHECK-NEXT:  1      2     1.00                        whilewr	p15.s, x30, x30
+# CHECK-NEXT:  1      1     1.00           *      U     wrffr	p0.b
+# CHECK-NEXT:  1      1     1.00           *      U     wrffr	p15.b
+# CHECK-NEXT:  1      4     1.00                        xar	z0.b, z0.b, z1.b, #1
+# CHECK-NEXT:  1      4     1.00                        xar	z0.d, z0.d, z1.d, #1
+# CHECK-NEXT:  1      4     1.00                        xar	z0.h, z0.h, z1.h, #1
+# CHECK-NEXT:  1      4     1.00                        xar	z0.s, z0.s, z1.s, #1
+# CHECK-NEXT:  1      4     1.00                        xar	z31.b, z31.b, z30.b, #8
+# CHECK-NEXT:  1      4     1.00                        xar	z31.d, z31.d, z30.d, #64
+# CHECK-NEXT:  1      4     1.00                        xar	z31.h, z31.h, z30.h, #16
+# CHECK-NEXT:  1      4     1.00                        xar	z31.s, z31.s, z30.s, #32
+# CHECK-NEXT:  1      2     1.00                        zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  1      2     1.00                        zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  1      2     1.00                        zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  1      2     1.00                        zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      2     1.00                        zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  1      2     1.00                        zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  1      2     1.00                        zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  1      2     1.00                        zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  1      2     1.00                        zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      2     1.00                        zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      2     1.00                        zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      2     1.00                        zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      3     1.00                        zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     1.00                        zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     1.00                        zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     1.00                        zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     1.00                        zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     1.00                        zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     1.00                        zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     1.00                        zip2	z31.s, z31.s, z31.s
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA320UnitALU
+# CHECK-NEXT: [1]   - CortexA320UnitB
+# CHECK-NEXT: [2]   - CortexA320UnitDiv
+# CHECK-NEXT: [3]   - CortexA320UnitLdSt
+# CHECK-NEXT: [4]   - CortexA320UnitMAC
+# CHECK-NEXT: [5]   - CortexA320UnitPAC
+# CHECK-NEXT: [6]   - CortexA320UnitVALU
+# CHECK-NEXT: [7]   - CortexA320UnitVMAC
+# CHECK-NEXT: [8]   - CortexA320UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]
+# CHECK-NEXT: 229.00 9.00    -     3876.00  -     -     2214.00 399.00 655.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]    Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     addp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addpl	sp, sp, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addpl	x0, x0, #-32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addpl	x21, x21, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addpl	x23, x8, #-1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addvl	sp, sp, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addvl	x0, x0, #-32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addvl	x21, x21, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     addvl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aesd	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aese	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aesimc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aesimc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aesmc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     aesmc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     andv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     andv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     andv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     andv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bcax	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  bdep	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     67.00  bdep	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     20.00  bdep	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     36.00  bdep	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  bext	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     67.00  bext	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     20.00  bext	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     36.00  bext	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00   1.00    -     bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00   1.00    -     bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00   1.00    -     bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00   1.00    -     bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  bgrp	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     67.00  bgrp	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     20.00  bgrp	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     36.00  bgrp	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bsl1n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     bsl2n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.d, z1.h, z15.h[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.d, z1.h, z31.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.d, z1.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.d, z1.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.d, z1.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.s, z1.b, z31.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z0.s, z1.b, z7.b[3], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z29.d, z30.h, z0.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z31.d, z30.h, z7.h[1], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cdot	z5.d, z6.h, z3.h[0], #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     cmla	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntb	x0, #28
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntd	x0, #28
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cnth	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cnth	x0, #28
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cnth	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cnth	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cntp	x0, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cntp	x0, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cntp	x0, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     cntp	x0, p15, p0.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntw	x0, #28
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     cntw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     compact	z31.d, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     compact	z31.s, p7, z31.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermeq	w30, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermeq	wzr, w30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermeq	x30, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermeq	xzr, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermne	w30, wzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermne	wzr, w30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermne	x30, xzr
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     ctermne	xzr, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decd	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     dech	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     dech	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     dech	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     dech	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     dech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     decp	z31.s, p15.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     decw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dupm	z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     dupm	z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eor3	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorbt	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorbt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorbt	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorbt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eortb	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eortb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eortb	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eortb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     eorv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ext	z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     5.00    -      -     fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     29.00   -      -     fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     13.00   -      -     fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     faddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -     faddv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     5.00    -      -     faddv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtlt	z0.s, p0/m, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtlt	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtx	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtx	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtxnt	z0.s, p0/m, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtxnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     10.00  fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     10.00  fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fexpa	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fexpa	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fexpa	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     flogb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     flogb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     flogb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlalt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmlslt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.d, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpe	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpe	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpe	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrte	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrte	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrte	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     19.00  fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     5.00   fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     9.00   fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     histcnt	z0.s, p0/z, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     histcnt	z29.d, p7/z, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     histseg	z0.b, z1.b, z31.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incd	z0.d, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     inch	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     inch	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     inch	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     inch	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     inch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     inch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     inch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incp	z31.s, p15.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     incw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     incw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z0.b, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z0.d, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z0.h, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z0.h, w0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z0.s, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z21.b, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z21.d, x10, x21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z21.s, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.b, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.b, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.d, #13, x8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.d, x13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.h, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.h, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.s, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z23.s, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.b, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.b, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.b, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.b, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.d, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.d, #-1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.d, xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.d, xzr, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.h, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.h, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.h, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.h, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.s, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.s, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.s, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     index	z31.s, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     insr	z31.b, b31
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z31.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     insr	z31.d, d31
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z31.d, xzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     insr	z31.h, h31
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z31.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     insr	z31.s, s31
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     insr	z31.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lasta	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lasta	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lasta	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lasta	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lasta	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lasta	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lasta	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lasta	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lastb	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lastb	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lastb	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lastb	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lastb	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lastb	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lastb	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     lastb	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     2.00    -      -      -      -      -     ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     3.00    -      -      -      -      -     ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1b	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1b	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1b	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1b	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1d	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1h	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1h	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1h	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1h	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sb	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sb	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sb	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sh	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sh	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1sh	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sw	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1w	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     ldnt1w	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     ldnt1w	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	p0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	z0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mad	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     match	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     match	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     match	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     match	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, b0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, p0/m, b0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, #32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, d0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, p0/m, d0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, p0/m, x0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, #32767
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, h0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, p0/m, h0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.q, q0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, #32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, p0/m, s0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, s0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.b, p7/m, b31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movprfx	z31, z6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.b, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.b, z31.b[63]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, p7/m, d31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, p7/m, sp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, sp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.d, z31.d[7]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.h, p7/m, h31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.h, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.h, z31.h[31]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.s, p7/m, s31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.s, wsp
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z31.s, z31.s[15]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, #-1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.d, #-6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.h, #-6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.q, z17.q[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     mov	z5.s, #-6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movs	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movs	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     msb	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     msb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     msb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     msb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     msr	ZCR_EL1, x3
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     msr	ZCR_EL12, x3
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     msr	ZCR_EL2, x3
+# CHECK-NEXT:  -     1.00    -      -      -      -      -      -      -     msr	ZCR_EL3, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     mul	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nbsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     nmatch	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     nmatch	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     nmatch	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     nmatch	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     orv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pfalse	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pmul	z29.b, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullb	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullt	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   pmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#7, p3, [z13.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	#7, p3, [z13.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1keep, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3strm, p5, [z10.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pldl3strm, p5, [z10.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfb	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#15, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#15, p7, [z31.s, #248]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfd	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#15, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#15, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfh	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#15, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#15, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	#7, p3, [x13, z8.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     prfw	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptest	p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptest	p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrue	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ptrues	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     punpkhi	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     punpkhi	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     punpklo	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     punpklo	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     raddhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   rax1	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffr	p0.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffr	p0.b, p0/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffr	p15.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffr	p15.b, p15/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffrs	p0.b, p0/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdffrs	p15.b, p15/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdvl	x0, #0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdvl	x21, #-32
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdvl	x23, #31
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     rdvl	xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev	z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rev	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     rsubhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     saba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     saba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     saba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     saba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     sadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     saddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sbclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sbclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sbclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sbclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     23.00  sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     23.00  sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     setffr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     shsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sli	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   sm4e	z0.s, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     1.00   sm4ekey	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smax	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     smin	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     smullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqcadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecp	z0.s, p0.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqdmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincp	z0.s, p0.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdcmlah	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlah	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmlsh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     sqrdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqrshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshlu	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sqxtunt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sri	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     srshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     srsra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sshllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubltb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubltb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubltb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ssubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -     8.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -     11.00   -      -      -      -      -     st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -     14.00   -      -      -      -      -     st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     14.00   -      -      -      -      -     st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     14.00   -      -      -      -      -     st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     14.00   -      -      -      -      -     st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     14.00   -      -      -      -      -     st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     25.00   -      -      -      -      -     st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -     50.00   -      -      -      -      -     st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1b	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1b	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1b	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1b	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1b	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1b	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1d	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1d	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1d	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1h	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1h	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1h	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1h	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1h	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1h	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1w	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1w	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1w	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -     7.00    -      -      -      -      -     stnt1w	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1w	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -     9.00    -      -      -      -      -     stnt1w	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	p0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	z0, [x0]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -     str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     suqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     tbx	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uaba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uaba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uaba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uaba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     uadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uaddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     23.00  udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     23.00  udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -     12.00  udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uhsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     umullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecp	z0.s, p0.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincb	x0, vl1
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincp	z0.s, p0.s
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	w0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	w0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	w0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	x0
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	x0, #14
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	x0, all, mul #16
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	x0, pow2
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     uqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     urecpe	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     urshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     ursqrte	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -     ursra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     ushllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -     usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     usubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.b, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.b, wzr, w0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.b, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.b, xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.d, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.d, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.h, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.h, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.s, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilege	p15.s, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilerw	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilerw	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilerw	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilerw	p15.s, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilewr	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilewr	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilewr	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     whilewr	p15.s, x30, x30
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     wrffr	p0.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -     wrffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z0.b, z0.b, z1.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z0.d, z0.d, z1.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z0.h, z0.h, z1.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z0.s, z0.s, z1.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z31.b, z31.b, z30.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z31.d, z31.d, z30.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z31.h, z31.h, z30.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     xar	z31.s, z31.s, z30.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -     zip2	z31.s, z31.s, z31.s
-- 
cgit v1.2.3


From a1c2a712939897251729b6fc436a2db7db6f03fc Mon Sep 17 00:00:00 2001
From: Andrei Golubev <andrey.golubev@intel.com>
Date: Wed, 18 Jun 2025 14:38:58 +0200
Subject: [mlir][bufferization] Use Type instead of Value in unknown conversion
 (#144658)

Generally, bufferization should be able to create a memref from a tensor
without needing to know more than just a mlir::Type. Thus, change
BufferizationOptions::UnknownTypeConverterFn to accept just a type
(mlir::TensorType for now) instead of mlir::Value. Additionally, apply
the same rationale to getMemRefType() helper function.

Both changes are prerequisites to enable custom types support in
one-shot bufferization.
---
 .../Bufferization/IR/BufferizableOpInterface.h        |  9 +++++----
 .../Bufferization/IR/BufferizableOpInterface.cpp      | 19 +++++++++----------
 .../Dialect/Bufferization/Transforms/Bufferize.cpp    |  4 ++--
 .../Transforms/SparsificationAndBufferizationPass.cpp |  6 +++---
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
index adccbef754ec..2fb795f16ae2 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
@@ -265,9 +265,9 @@ struct BufferizationOptions {
       std::function<BaseMemRefType(TensorType, Attribute memorySpace,
                                    func::FuncOp, const BufferizationOptions &)>;
   /// Tensor -> MemRef type converter.
-  /// Parameters: Value, memory space, bufferization options
+  /// Parameters: tensor type, memory space, bufferization options
   using UnknownTypeConverterFn = std::function<BaseMemRefType(
-      Value, Attribute memorySpace, const BufferizationOptions &)>;
+      TensorType, Attribute memorySpace, const BufferizationOptions &)>;
   // Produce a MemorySpace attribute from a tensor type
   using DefaultMemorySpaceFn =
       std::function<std::optional<Attribute>(TensorType t)>;
@@ -655,7 +655,7 @@ OpTy replaceOpWithNewBufferizedOp(RewriterBase &rewriter, Operation *op,
   return newOp;
 }
 
-/// Return a MemRefType to which the type of the given value can be bufferized.
+/// Return a MemRefType to which the TensorType can be bufferized.
 ///
 /// If possible, op bufferization implementations should not use this function
 /// and instead infer precise memref types for tensor results by themselves.
@@ -667,7 +667,8 @@ OpTy replaceOpWithNewBufferizedOp(RewriterBase &rewriter, Operation *op,
 /// Note: Canonicalization patterns could clean up layout maps and infer more
 /// precise layout maps after bufferization. However, many possible
 /// canonicalizations are currently not implemented.
-BaseMemRefType getMemRefType(Value value, const BufferizationOptions &options,
+BaseMemRefType getMemRefType(TensorType tensorType,
+                             const BufferizationOptions &options,
                              MemRefLayoutAttrInterface layout = {},
                              Attribute memorySpace = nullptr);
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index 1d6e1bdaf80f..dd43647682ea 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -345,10 +345,9 @@ defaultFunctionArgTypeConverter(TensorType type, Attribute memorySpace,
 }
 /// Default unknown type converter: Use a fully dynamic layout map.
 BaseMemRefType
-defaultUnknownTypeConverter(Value value, Attribute memorySpace,
+defaultUnknownTypeConverter(TensorType tensorType, Attribute memorySpace,
                             const BufferizationOptions &options) {
-  return getMemRefTypeWithFullyDynamicLayout(
-      llvm::cast<TensorType>(value.getType()), memorySpace);
+  return getMemRefTypeWithFullyDynamicLayout(tensorType, memorySpace);
 }
 
 } // namespace
@@ -724,7 +723,8 @@ bufferization::getBufferType(Value value, const BufferizationOptions &options,
   if (!memSpace.has_value())
     return op->emitError("could not infer memory space");
 
-  return getMemRefType(value, options, /*layout=*/{}, *memSpace);
+  return getMemRefType(cast<TensorType>(value.getType()), options,
+                       /*layout=*/{}, *memSpace);
 }
 
 bool bufferization::hasTensorSemantics(Operation *op) {
@@ -797,12 +797,10 @@ LogicalResult BufferizationOptions::createMemCpy(OpBuilder &b, Location loc,
 // Bufferization-specific IRMapping support with debugging.
 //===----------------------------------------------------------------------===//
 
-BaseMemRefType bufferization::getMemRefType(Value value,
+BaseMemRefType bufferization::getMemRefType(TensorType tensorType,
                                             const BufferizationOptions &options,
                                             MemRefLayoutAttrInterface layout,
                                             Attribute memorySpace) {
-  auto tensorType = llvm::cast<TensorType>(value.getType());
-
   // Case 1: Unranked memref type.
   if (auto unrankedTensorType =
           llvm::dyn_cast<UnrankedTensorType>(tensorType)) {
@@ -819,7 +817,7 @@ BaseMemRefType bufferization::getMemRefType(Value value,
                            memorySpace);
   }
 
-  return options.unknownTypeConverterFn(value, memorySpace, options);
+  return options.unknownTypeConverterFn(tensorType, memorySpace, options);
 }
 
 BaseMemRefType
@@ -955,10 +953,11 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
     const BufferizationState &bufferizationState,
     SmallVector<Value> &invocationStack) {
   assert(llvm::isa<TensorType>(value.getType()) && "expected tensor type");
+  auto tensorType = cast<TensorType>(value.getType());
 
   // No further analysis is possible for a block argument.
   if (llvm::isa<BlockArgument>(value))
-    return bufferization::getMemRefType(value, options);
+    return bufferization::getMemRefType(tensorType, options);
 
   // Value is an OpResult.
   Operation *op = getOwnerOfValue(value);
@@ -981,7 +980,7 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
   if (!memSpace.has_value())
     return op->emitError("could not infer memory space");
 
-  return getMemRefType(value, options, /*layout=*/{}, *memSpace);
+  return getMemRefType(tensorType, options, /*layout=*/{}, *memSpace);
 }
 
 bool bufferization::detail::defaultIsRepetitiveRegion(
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index c7681d309a4a..7e9b9119ce94 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -109,9 +109,9 @@ struct OneShotBufferizePass
                   "'unknown-type-conversion'");
         return signalPassFailure();
       }
-      opt.unknownTypeConverterFn = [=](Value value, Attribute memorySpace,
+      opt.unknownTypeConverterFn = [=](TensorType tensorType,
+                                       Attribute memorySpace,
                                        const BufferizationOptions &options) {
-        auto tensorType = cast<TensorType>(value.getType());
         if (unknownTypeConversionOption == LayoutMapOption::IdentityLayoutMap)
           return bufferization::getMemRefTypeWithStaticIdentityLayout(
               tensorType, memorySpace);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
index a3ab53d81811..15e5102462ad 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparsificationAndBufferizationPass.cpp
@@ -223,10 +223,10 @@ mlir::getBufferizationOptionsForSparsification(bool analysisOnly) {
   OneShotBufferizationOptions options;
   options.bufferizeFunctionBoundaries = true;
   options.setFunctionBoundaryTypeConversion(LayoutMapOption::IdentityLayoutMap);
-  options.unknownTypeConverterFn = [](Value value, Attribute memorySpace,
+  options.unknownTypeConverterFn = [](TensorType tensorType,
+                                      Attribute memorySpace,
                                       const BufferizationOptions &options) {
-    return getMemRefTypeWithStaticIdentityLayout(
-        cast<TensorType>(value.getType()), memorySpace);
+    return getMemRefTypeWithStaticIdentityLayout(tensorType, memorySpace);
   };
   if (analysisOnly) {
     options.testAnalysisOnly = true;
-- 
cgit v1.2.3


From 66580f77b826e71a9727f1d6287bec6a6101f620 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Wed, 18 Jun 2025 14:42:09 +0200
Subject: [mlir][Transforms][NFC] Dialect Conversion: Keep
 `unresolvedMaterializations` up to date (#144254)

`unresolvedMaterializations` is a mapping from
`UnrealizedConversionCastOp` to `UnresolvedMaterializationRewrite`. This
mapping is needed to find the correct type converter for an unresolved
materialization.

With this commit, `unresolvedMaterializations` is updated immediately
when an op is being erased. This also cleans up the code base a bit:
`SingleEraseRewriter` is now used only during the "cleanup" phase and no
longer needed as a field of `ConversionRewriterImpl`.

This commit is in preparation of the One-Shot Dialect Conversion
refactoring: `allowPatternRollback = false` will in the future trigger
immediate materialization of all IR changes.
---
 mlir/lib/Transforms/Utils/DialectConversion.cpp | 33 +++++++++++++++----------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index 7de26d7cfa84..c4b85ec4f67d 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -848,7 +848,7 @@ namespace detail {
 struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   explicit ConversionPatternRewriterImpl(MLIRContext *ctx,
                                          const ConversionConfig &config)
-      : context(ctx), eraseRewriter(ctx), config(config) {}
+      : context(ctx), config(config) {}
 
   //===--------------------------------------------------------------------===//
   // State Management
@@ -981,8 +981,11 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// no new IR is created between calls to `eraseOp`/`eraseBlock`.
   struct SingleEraseRewriter : public RewriterBase, RewriterBase::Listener {
   public:
-    SingleEraseRewriter(MLIRContext *context)
-        : RewriterBase(context, /*listener=*/this) {}
+    SingleEraseRewriter(
+        MLIRContext *context,
+        std::function<void(Operation *)> opErasedCallback = nullptr)
+        : RewriterBase(context, /*listener=*/this),
+          opErasedCallback(opErasedCallback) {}
 
     /// Erase the given op (unless it was already erased).
     void eraseOp(Operation *op) override {
@@ -1003,13 +1006,20 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
 
     bool wasErased(void *ptr) const { return erased.contains(ptr); }
 
-    void notifyOperationErased(Operation *op) override { erased.insert(op); }
+    void notifyOperationErased(Operation *op) override {
+      erased.insert(op);
+      if (opErasedCallback)
+        opErasedCallback(op);
+    }
 
     void notifyBlockErased(Block *block) override { erased.insert(block); }
 
   private:
     /// Pointers to all erased operations and blocks.
     DenseSet<void *> erased;
+
+    /// A callback that is invoked when an operation is erased.
+    std::function<void(Operation *)> opErasedCallback;
   };
 
   //===--------------------------------------------------------------------===//
@@ -1019,11 +1029,6 @@ struct ConversionPatternRewriterImpl : public RewriterBase::Listener {
   /// MLIR context.
   MLIRContext *context;
 
-  /// A rewriter that keeps track of ops/block that were already erased and
-  /// skips duplicate op/block erasures. This rewriter is used during the
-  /// "cleanup" phase.
-  SingleEraseRewriter eraseRewriter;
-
   // Mapping between replaced values that differ in type. This happens when
   // replacing a value with one of a different type.
   ConversionValueMapping mapping;
@@ -1195,6 +1200,11 @@ void ConversionPatternRewriterImpl::applyRewrites() {
     rewrites[i]->commit(rewriter);
 
   // Clean up all rewrites.
+  SingleEraseRewriter eraseRewriter(
+      context, /*opErasedCallback=*/[&](Operation *op) {
+        if (auto castOp = dyn_cast<UnrealizedConversionCastOp>(op))
+          unresolvedMaterializations.erase(castOp);
+      });
   for (auto &rewrite : rewrites)
     rewrite->cleanup(eraseRewriter);
 }
@@ -2714,11 +2724,8 @@ LogicalResult OperationConverter::convertOperations(ArrayRef<Operation *> ops) {
   SmallVector<UnrealizedConversionCastOp> allCastOps;
   const DenseMap<UnrealizedConversionCastOp, UnresolvedMaterializationRewrite *>
       &materializations = rewriterImpl.unresolvedMaterializations;
-  for (auto it : materializations) {
-    if (rewriterImpl.eraseRewriter.wasErased(it.first))
-      continue;
+  for (auto it : materializations)
     allCastOps.push_back(it.first);
-  }
 
   // Reconcile all UnrealizedConversionCastOps that were inserted by the
   // dialect conversion frameworks. (Not the one that were inserted by
-- 
cgit v1.2.3


From 4b2ab1494bc07493087252dff4e5e19808703048 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 18 Jun 2025 07:46:49 -0500
Subject: [flang][OpenMP] Don't crash on iterator modifier in declare mapper
 (#144359)

Both the declare mapper directive argument, and the iterator modifier
can contain declaration-type-spec, so make sure that the processing of
one ends before processing of the other begins in semantic analysis.
---
 flang/lib/Semantics/resolve-names.cpp                    |  2 +-
 flang/test/Lower/OpenMP/Todo/declare-mapper-iterator.f90 | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/declare-mapper-iterator.f90

diff --git a/flang/lib/Semantics/resolve-names.cpp b/flang/lib/Semantics/resolve-names.cpp
index f66918e5c140..9e465f8ff3e1 100644
--- a/flang/lib/Semantics/resolve-names.cpp
+++ b/flang/lib/Semantics/resolve-names.cpp
@@ -1800,9 +1800,9 @@ void OmpVisitor::ProcessMapperSpecifier(const parser::OmpMapperSpecifier &spec,
   Walk(std::get<parser::TypeSpec>(spec.t));
   auto &varName{std::get<parser::Name>(spec.t)};
   DeclareObjectEntity(varName);
+  EndDeclTypeSpec();
 
   Walk(clauses);
-  EndDeclTypeSpec();
   PopScope();
 }
 
diff --git a/flang/test/Lower/OpenMP/Todo/declare-mapper-iterator.f90 b/flang/test/Lower/OpenMP/Todo/declare-mapper-iterator.f90
new file mode 100644
index 000000000000..dacd6d624659
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/declare-mapper-iterator.f90
@@ -0,0 +1,11 @@
+!RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 -o - %s 2>&1 | FileCheck %s
+
+!CHECK: Support for iterator modifiers is not implemented yet
+subroutine f(arg)
+  type :: s
+    integer :: a(10)
+  end type
+  type(s) :: arg(:)
+
+  !$omp declare mapper(m: s :: v) map(mapper(m), iterator(i = 1:10): v%a(i))
+end
-- 
cgit v1.2.3


From a83d3362f686725bac76bfb9562663908de25f15 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Wed, 18 Jun 2025 14:02:11 +0100
Subject: [flang][OpenMP] Don't allow DO CONCURRENT inside of a loop nest
 (#144506)

I don't think DO CONCURRENT fits the definition of a Canonical Loop Nest
(OpenMP 6.0 section 6.4.1).
It is however explicitly allowed for the LOOP construct (6.0 section
13.8).

There's some obscure language in OpenMP 6.0 for the LOOP construct:

> If the collapsed loop is a DO CONCURRENT loop, neither the
> data-sharing attribute clauses nor the collapse clause may be
specified.

From the surrounding context, I think "collapsed loop" just means the
loop that the LOOP construct applies to. So I will interpret this to
mean that DO CONCURRENT can only be used with the LOOP construct if it
does not contain the COLLAPSE clause.

This also fixes a bug where the associated clause was never cleared
after it was set.

Fixes #144178
---
 flang/lib/Semantics/resolve-directives.cpp         | 38 +++++++++++++++++----
 flang/test/Lower/OpenMP/Todo/omp-doconcurrent.f90  | 10 ------
 .../Semantics/OpenMP/do-concurrent-collapse.f90    | 39 ++++++++++++++++++++++
 3 files changed, 71 insertions(+), 16 deletions(-)
 delete mode 100644 flang/test/Lower/OpenMP/Todo/omp-doconcurrent.f90
 create mode 100644 flang/test/Semantics/OpenMP/do-concurrent-collapse.f90

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 282660684e78..57db76e2160d 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -23,6 +23,7 @@
 #include "flang/Semantics/openmp-modifiers.h"
 #include "flang/Semantics/symbol.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
 #include "llvm/Support/Debug.h"
 #include <list>
 #include <map>
@@ -740,9 +741,7 @@ public:
   }
 
   const parser::OmpClause *associatedClause{nullptr};
-  void SetAssociatedClause(const parser::OmpClause &c) {
-    associatedClause = &c;
-  }
+  void SetAssociatedClause(const parser::OmpClause *c) { associatedClause = c; }
   const parser::OmpClause *GetAssociatedClause() { return associatedClause; }
 
 private:
@@ -1919,12 +1918,17 @@ std::int64_t OmpAttributeVisitor::GetAssociatedLoopLevelFromClauses(
   }
 
   if (orderedLevel && (!collapseLevel || orderedLevel >= collapseLevel)) {
-    SetAssociatedClause(*ordClause);
+    SetAssociatedClause(ordClause);
     return orderedLevel;
   } else if (!orderedLevel && collapseLevel) {
-    SetAssociatedClause(*collClause);
+    SetAssociatedClause(collClause);
     return collapseLevel;
-  } // orderedLevel < collapseLevel is an error handled in structural checks
+  } else {
+    SetAssociatedClause(nullptr);
+  }
+  // orderedLevel < collapseLevel is an error handled in structural
+  // checks
+
   return 1; // default is outermost loop
 }
 
@@ -1952,9 +1956,31 @@ void OmpAttributeVisitor::PrivatizeAssociatedLoopIndexAndCheckLoopLevel(
     ivDSA = Symbol::Flag::OmpLastPrivate;
   }
 
+  bool isLoopConstruct{
+      GetContext().directive == llvm::omp::Directive::OMPD_loop};
+  const parser::OmpClause *clause{GetAssociatedClause()};
+  bool hasCollapseClause{
+      clause ? (clause->Id() == llvm::omp::OMPC_collapse) : false};
+
   const auto &outer{std::get<std::optional<parser::DoConstruct>>(x.t)};
   if (outer.has_value()) {
     for (const parser::DoConstruct *loop{&*outer}; loop && level > 0; --level) {
+      if (loop->IsDoConcurrent()) {
+        // DO CONCURRENT is explicitly allowed for the LOOP construct so long as
+        // there isn't a COLLAPSE clause
+        if (isLoopConstruct) {
+          if (hasCollapseClause) {
+            // hasCollapseClause implies clause != nullptr
+            context_.Say(clause->source,
+                "DO CONCURRENT loops cannot be used with the COLLAPSE clause."_err_en_US);
+          }
+        } else {
+          auto &stmt =
+              std::get<parser::Statement<parser::NonLabelDoStmt>>(loop->t);
+          context_.Say(stmt.source,
+              "DO CONCURRENT loops cannot form part of a loop nest."_err_en_US);
+        }
+      }
       // go through all the nested do-loops and resolve index variables
       const parser::Name *iv{GetLoopIndex(*loop)};
       if (iv) {
diff --git a/flang/test/Lower/OpenMP/Todo/omp-doconcurrent.f90 b/flang/test/Lower/OpenMP/Todo/omp-doconcurrent.f90
deleted file mode 100644
index a6d70fa44592..000000000000
--- a/flang/test/Lower/OpenMP/Todo/omp-doconcurrent.f90
+++ /dev/null
@@ -1,10 +0,0 @@
-! RUN: %not_todo_cmd bbc -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-! RUN: %not_todo_cmd %flang_fc1 -emit-fir -fopenmp -o - %s 2>&1 | FileCheck %s
-
-! CHECK: not yet implemented: Do Concurrent in Worksharing loop construct
-subroutine sb()
-  !$omp do
-  do concurrent(i=1:10)
-    print *, i
-  end do
-end subroutine
diff --git a/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90 b/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90
new file mode 100644
index 000000000000..bb1929249183
--- /dev/null
+++ b/flang/test/Semantics/OpenMP/do-concurrent-collapse.f90
@@ -0,0 +1,39 @@
+!RUN: %python %S/../test_errors.py %s %flang -fopenmp
+
+integer :: i, j
+!$omp parallel do collapse(2)
+do i = 1, 1
+  ! ERROR: DO CONCURRENT loops cannot form part of a loop nest.
+  do concurrent (j = 1:2)
+    print *, j
+  end do
+end do
+
+!$omp parallel do
+do i = 1, 1
+  ! This should not lead to an error because it is not part of a loop nest:
+  do concurrent (j = 1:2)
+    print *, j
+  end do
+end do
+
+!$omp parallel do
+! ERROR: DO CONCURRENT loops cannot form part of a loop nest.
+do concurrent (j = 1:2)
+  print *, j
+end do
+
+!$omp loop
+! Do concurrent is explicitly allowed inside of omp loop
+do concurrent (j = 1:2)
+  print *, j
+end do
+
+! ERROR: DO CONCURRENT loops cannot be used with the COLLAPSE clause.
+!$omp loop collapse(2)
+do i = 1, 1
+  do concurrent (j = 1:2)
+    print *, j
+  end do
+end do
+end
-- 
cgit v1.2.3


From 8584abb05a84d3bf4e84cdfe4154d7ade8bdfd04 Mon Sep 17 00:00:00 2001
From: Frank Schlimbach <frank.schlimbach@intel.com>
Date: Wed, 18 Jun 2025 15:04:55 +0200
Subject: [mlir] mlir/test/lit.local.cfg ->
 mlir/test/Target/SPIRV/lit.local.cfg (#144685)

renamed: mlir/test/lit.local.cfg -> mlir/test/Target/SPIRV/lit.local.cfg
---
 mlir/test/Target/SPIRV/lit.local.cfg | 7 +++++++
 mlir/test/lit.local.cfg              | 7 -------
 2 files changed, 7 insertions(+), 7 deletions(-)
 create mode 100644 mlir/test/Target/SPIRV/lit.local.cfg
 delete mode 100644 mlir/test/lit.local.cfg

diff --git a/mlir/test/Target/SPIRV/lit.local.cfg b/mlir/test/Target/SPIRV/lit.local.cfg
new file mode 100644
index 000000000000..167c454db518
--- /dev/null
+++ b/mlir/test/Target/SPIRV/lit.local.cfg
@@ -0,0 +1,7 @@
+if not "SPIRV" in config.root.targets:
+    config.unsupported = True
+
+if config.spirv_tools_tests:
+    config.available_features.add("spirv-tools")
+    config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
+    config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
diff --git a/mlir/test/lit.local.cfg b/mlir/test/lit.local.cfg
deleted file mode 100644
index 167c454db518..000000000000
--- a/mlir/test/lit.local.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-if not "SPIRV" in config.root.targets:
-    config.unsupported = True
-
-if config.spirv_tools_tests:
-    config.available_features.add("spirv-tools")
-    config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
-    config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
-- 
cgit v1.2.3


From 68471d29eed2c49f9b439e505b3f24d387d54f97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder@redhat.com>
Date: Wed, 18 Jun 2025 15:17:53 +0200
Subject: =?UTF-8?q?Revert=20"Reapply=20"[clang][bytecode]=20Allocate=20Int?=
 =?UTF-8?q?egralAP=20and=20Floating=20types=20usi=E2=80=A6=20(#144676)"?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 7c15edb306932e41c159f3d69c161ed0d89d47b7.

This still breaks clang-armv8-quick:
https://lab.llvm.org/buildbot/#/builders/154/builds/17587
---
 clang/lib/AST/ByteCode/Compiler.cpp                | 112 +++----
 clang/lib/AST/ByteCode/Compiler.h                  |   1 -
 clang/lib/AST/ByteCode/Descriptor.cpp              |   2 +-
 clang/lib/AST/ByteCode/Disasm.cpp                  |  60 ++--
 clang/lib/AST/ByteCode/Floating.h                  | 252 +++++----------
 clang/lib/AST/ByteCode/Integral.h                  |   3 -
 clang/lib/AST/ByteCode/IntegralAP.h                | 234 ++++++--------
 clang/lib/AST/ByteCode/Interp.cpp                  | 106 +------
 clang/lib/AST/ByteCode/Interp.h                    | 337 ++++-----------------
 clang/lib/AST/ByteCode/InterpBuiltin.cpp           |  55 +---
 clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp    |   4 +-
 clang/lib/AST/ByteCode/InterpState.h               |  30 --
 clang/lib/AST/ByteCode/Opcodes.td                  |  14 +-
 clang/lib/AST/ByteCode/PrimType.h                  |  17 --
 clang/lib/AST/ByteCode/Program.h                   |  24 +-
 .../AST/ByteCode/builtin-bit-cast-long-double.cpp  |  10 +-
 clang/test/AST/ByteCode/builtin-functions.cpp      |  12 +-
 17 files changed, 343 insertions(+), 930 deletions(-)

diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 3f884ed8d094..9fe4803ce98e 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -748,8 +748,7 @@ bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
   if (DiscardResult)
     return true;
 
-  APFloat F = E->getValue();
-  return this->emitFloat(F, E);
+  return this->emitConstFloat(E->getValue(), E);
 }
 
 template <class Emitter>
@@ -4186,10 +4185,8 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
                              nullptr, E);
   case PT_MemberPtr:
     return this->emitNullMemberPtr(0, nullptr, E);
-  case PT_Float: {
-    APFloat F = APFloat::getZero(Ctx.getFloatSemantics(QT));
-    return this->emitFloat(F, E);
-  }
+  case PT_Float:
+    return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
   case PT_FixedPoint: {
     auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
     return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
@@ -4677,7 +4674,10 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       if (!visitInitializer(Init))
         return false;
 
-      return this->emitFinishInitGlobal(Init);
+      if (!this->emitFinishInit(Init))
+        return false;
+
+      return this->emitPopPtr(Init);
     };
 
     DeclScope<Emitter> LocalScope(this, VD);
@@ -4698,45 +4698,51 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       return false;
 
     return !Init || (checkDecl() && initGlobal(*GlobalIndex));
-  }
-  // Local variables.
-  InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
+  } else {
+    InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
 
-  if (VarT) {
-    unsigned Offset = this->allocateLocalPrimitive(
-        VD, *VarT, VD->getType().isConstQualified(), nullptr, ScopeKind::Block,
-        IsConstexprUnknown);
-    if (Init) {
-      // If this is a toplevel declaration, create a scope for the
-      // initializer.
-      if (Toplevel) {
-        LocalScope<Emitter> Scope(this);
-        if (!this->visit(Init))
-          return false;
-        return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
-      } else {
-        if (!this->visit(Init))
-          return false;
-        return this->emitSetLocal(*VarT, Offset, VD);
+    if (VarT) {
+      unsigned Offset = this->allocateLocalPrimitive(
+          VD, *VarT, VD->getType().isConstQualified(), nullptr,
+          ScopeKind::Block, IsConstexprUnknown);
+      if (Init) {
+        // If this is a toplevel declaration, create a scope for the
+        // initializer.
+        if (Toplevel) {
+          LocalScope<Emitter> Scope(this);
+          if (!this->visit(Init))
+            return false;
+          return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
+        } else {
+          if (!this->visit(Init))
+            return false;
+          return this->emitSetLocal(*VarT, Offset, VD);
+        }
       }
-    }
-  } else {
-    if (std::optional<unsigned> Offset = this->allocateLocal(
-            VD, VD->getType(), nullptr, ScopeKind::Block, IsConstexprUnknown)) {
-      if (!Init)
-        return true;
+    } else {
+      if (std::optional<unsigned> Offset =
+              this->allocateLocal(VD, VD->getType(), nullptr, ScopeKind::Block,
+                                  IsConstexprUnknown)) {
+        if (!Init)
+          return true;
 
-      if (!this->emitGetPtrLocal(*Offset, Init))
-        return false;
+        if (!this->emitGetPtrLocal(*Offset, Init))
+          return false;
 
-      if (!visitInitializer(Init))
-        return false;
+        if (!visitInitializer(Init))
+          return false;
+
+        if (!this->emitFinishInit(Init))
+          return false;
 
-      return this->emitFinishInitPop(Init);
+        return this->emitPopPtr(Init);
+      }
+      return false;
     }
-    return false;
+    return true;
   }
-  return true;
+
+  return false;
 }
 
 template <class Emitter>
@@ -4745,10 +4751,8 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
   assert(!DiscardResult);
   if (Val.isInt())
     return this->emitConst(Val.getInt(), ValType, E);
-  else if (Val.isFloat()) {
-    APFloat F = Val.getFloat();
-    return this->emitFloat(F, E);
-  }
+  else if (Val.isFloat())
+    return this->emitConstFloat(Val.getFloat(), E);
 
   if (Val.isLValue()) {
     if (Val.isNullPointer())
@@ -6129,10 +6133,8 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      APFloat F(TargetSemantics, 1);
-      if (!this->emitFloat(F, E))
+      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
         return false;
-
       if (!this->emitAddf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6174,10 +6176,8 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      APFloat F(TargetSemantics, 1);
-      if (!this->emitFloat(F, E))
+      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
         return false;
-
       if (!this->emitSubf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6953,20 +6953,6 @@ bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
   return true;
 }
 
-template <class Emitter>
-bool Compiler<Emitter>::emitFloat(const APFloat &F, const Expr *E) {
-  assert(!DiscardResult && "Should've been checked before");
-
-  if (Floating::singleWord(F.getSemantics()))
-    return this->emitConstFloat(Floating(F), E);
-
-  APInt I = F.bitcastToAPInt();
-  return this->emitConstFloat(
-      Floating(const_cast<uint64_t *>(I.getRawData()),
-               llvm::APFloatBase::SemanticsToEnum(F.getSemantics())),
-      E);
-}
-
 //  This function is constexpr if and only if To, From, and the types of
 //  all subobjects of To and From are types T such that...
 //  (3.1) - is_union_v<T> is false;
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index a1d068cc7e0a..ac3ad84766dc 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -391,7 +391,6 @@ private:
   bool emitRecordDestruction(const Record *R, SourceInfo Loc);
   bool emitDestruction(const Descriptor *Desc, SourceInfo Loc);
   bool emitDummyPtr(const DeclTy &D, const Expr *E);
-  bool emitFloat(const APFloat &F, const Expr *E);
   unsigned collectBaseOffset(const QualType BaseType,
                              const QualType DerivedType);
   bool emitLambdaStaticInvokerBody(const CXXMethodDecl *MD);
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 46e4d0d940b3..5531295dfa2f 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -368,7 +368,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsTemporary, bool IsConst, UnknownSize)
     : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
       MDSize(MD.value_or(0)),
-      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
+      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)),
       IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
       IsArray(true), CtorFn(getCtorArrayPrim(Type)),
       DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 7c6b78386b14..846dc2fe92a7 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -50,56 +50,34 @@ inline static std::string printArg(Program &P, CodePtr &OpPC) {
 }
 
 template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
-  auto Sem = Floating::deserializeSemantics(*OpPC);
-
-  unsigned BitWidth = llvm::APFloatBase::semanticsSizeInBits(
-      llvm::APFloatBase::EnumToSemantics(Sem));
-  auto Memory =
-      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
-  Floating Result(Memory.get(), Sem);
-  Floating::deserialize(*OpPC, &Result);
-
-  OpPC += align(Result.bytesToSerialize());
+  auto F = Floating::deserialize(*OpPC);
+  OpPC += align(F.bytesToSerialize());
 
-  std::string S;
-  llvm::raw_string_ostream SS(S);
-  SS << Result;
-  return S;
+  std::string Result;
+  llvm::raw_string_ostream SS(Result);
+  SS << F;
+  return Result;
 }
 
 template <>
 inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
-  using T = IntegralAP<false>;
-  unsigned BitWidth = T::deserializeSize(*OpPC);
-  auto Memory =
-      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
-
-  T Result(Memory.get(), BitWidth);
-  T::deserialize(*OpPC, &Result);
-
-  OpPC += Result.bytesToSerialize();
-  std::string Str;
-  llvm::raw_string_ostream SS(Str);
-  SS << Result;
-  return Str;
-}
+  auto F = IntegralAP<false>::deserialize(*OpPC);
+  OpPC += align(F.bytesToSerialize());
 
+  std::string Result;
+  llvm::raw_string_ostream SS(Result);
+  SS << F;
+  return Result;
+}
 template <>
 inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
-  using T = IntegralAP<true>;
-  unsigned BitWidth = T::deserializeSize(*OpPC);
-  auto Memory =
-      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
-
-  T Result(Memory.get(), BitWidth);
-  T::deserialize(*OpPC, &Result);
-
-  std::string Str;
-  llvm::raw_string_ostream SS(Str);
-  SS << Result;
+  auto F = IntegralAP<true>::deserialize(*OpPC);
+  OpPC += align(F.bytesToSerialize());
 
-  OpPC += Result.bytesToSerialize();
-  return Str;
+  std::string Result;
+  llvm::raw_string_ostream SS(Result);
+  SS << F;
+  return Result;
 }
 
 template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {
diff --git a/clang/lib/AST/ByteCode/Floating.h b/clang/lib/AST/ByteCode/Floating.h
index 659892e720ab..3750568fc23c 100644
--- a/clang/lib/AST/ByteCode/Floating.h
+++ b/clang/lib/AST/ByteCode/Floating.h
@@ -17,79 +17,63 @@
 #include "clang/AST/APValue.h"
 #include "llvm/ADT/APFloat.h"
 
-// XXX This is just a debugging help. Setting this to 1 will heap-allocate ALL
-// floating values.
-#define ALLOCATE_ALL 0
-
 namespace clang {
 namespace interp {
 
 using APFloat = llvm::APFloat;
 using APSInt = llvm::APSInt;
-using APInt = llvm::APInt;
 
-/// If a Floating is constructed from Memory, it DOES NOT OWN THAT MEMORY.
-/// It will NOT copy the memory (unless, of course, copy() is called) and it
-/// won't alllocate anything. The allocation should happen via InterpState or
-/// Program.
 class Floating final {
 private:
-  union {
-    uint64_t Val = 0;
-    uint64_t *Memory;
-  };
-  llvm::APFloatBase::Semantics Semantics;
-
-  APFloat getValue() const {
-    unsigned BitWidth = bitWidth();
-    if (singleWord())
-      return APFloat(getSemantics(), APInt(BitWidth, Val));
-    unsigned NumWords = numWords();
-    return APFloat(getSemantics(), APInt(BitWidth, NumWords, Memory));
-  }
+  // The underlying value storage.
+  APFloat F;
 
 public:
-  Floating() = default;
-  Floating(llvm::APFloatBase::Semantics Semantics)
-      : Val(0), Semantics(Semantics) {}
-  Floating(const APFloat &F) {
+  /// Zero-initializes a Floating.
+  Floating() : F(0.0f) {}
+  Floating(const APFloat &F) : F(F) {}
 
-    Semantics = llvm::APFloatBase::SemanticsToEnum(F.getSemantics());
-    this->copy(F);
+  // Static constructors for special floating point values.
+  static Floating getInf(const llvm::fltSemantics &Sem) {
+    return Floating(APFloat::getInf(Sem));
   }
-  Floating(uint64_t *Memory, llvm::APFloatBase::Semantics Semantics)
-      : Memory(Memory), Semantics(Semantics) {}
-
-  APFloat getAPFloat() const { return getValue(); }
+  const APFloat &getAPFloat() const { return F; }
 
-  bool operator<(Floating RHS) const { return getValue() < RHS.getValue(); }
-  bool operator>(Floating RHS) const { return getValue() > RHS.getValue(); }
-  bool operator<=(Floating RHS) const { return getValue() <= RHS.getValue(); }
-  bool operator>=(Floating RHS) const { return getValue() >= RHS.getValue(); }
+  bool operator<(Floating RHS) const { return F < RHS.F; }
+  bool operator>(Floating RHS) const { return F > RHS.F; }
+  bool operator<=(Floating RHS) const { return F <= RHS.F; }
+  bool operator>=(Floating RHS) const { return F >= RHS.F; }
+  bool operator==(Floating RHS) const { return F == RHS.F; }
+  bool operator!=(Floating RHS) const { return F != RHS.F; }
+  Floating operator-() const { return Floating(-F); }
 
   APFloat::opStatus convertToInteger(APSInt &Result) const {
     bool IsExact;
-    return getValue().convertToInteger(Result, llvm::APFloat::rmTowardZero,
-                                       &IsExact);
+    return F.convertToInteger(Result, llvm::APFloat::rmTowardZero, &IsExact);
   }
 
-  void toSemantics(const llvm::fltSemantics *Sem, llvm::RoundingMode RM,
-                   Floating *Result) const {
-    APFloat Copy = getValue();
+  Floating toSemantics(const llvm::fltSemantics *Sem,
+                       llvm::RoundingMode RM) const {
+    APFloat Copy = F;
     bool LosesInfo;
     Copy.convert(*Sem, RM, &LosesInfo);
     (void)LosesInfo;
-    Result->copy(Copy);
+    return Floating(Copy);
+  }
+
+  /// Convert this Floating to one with the same semantics as \Other.
+  Floating toSemantics(const Floating &Other, llvm::RoundingMode RM) const {
+    return toSemantics(&Other.F.getSemantics(), RM);
   }
 
   APSInt toAPSInt(unsigned NumBits = 0) const {
-    return APSInt(getValue().bitcastToAPInt());
+    return APSInt(F.bitcastToAPInt());
   }
-  APValue toAPValue(const ASTContext &) const { return APValue(getValue()); }
+  APValue toAPValue(const ASTContext &) const { return APValue(F); }
   void print(llvm::raw_ostream &OS) const {
     // Can't use APFloat::print() since it appends a newline.
     SmallVector<char, 16> Buffer;
-    getValue().toString(Buffer);
+    F.toString(Buffer);
     OS << Buffer;
   }
   std::string toDiagnosticString(const ASTContext &Ctx) const {
@@ -99,62 +83,25 @@ public:
     return NameStr;
   }
 
-  unsigned bitWidth() const {
-    return llvm::APFloatBase::semanticsSizeInBits(getSemantics());
-  }
-  unsigned numWords() const { return llvm::APInt::getNumWords(bitWidth()); }
-  bool singleWord() const {
-#if ALLOCATE_ALL
-    return false;
-#endif
-    return numWords() == 1;
-  }
-  static bool singleWord(const llvm::fltSemantics &Sem) {
-#if ALLOCATE_ALL
-    return false;
-#endif
-    return APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem)) == 1;
-  }
-  const llvm::fltSemantics &getSemantics() const {
-    return llvm::APFloatBase::EnumToSemantics(Semantics);
-  }
-
-  void copy(const APFloat &F) {
-    if (singleWord()) {
-      Val = F.bitcastToAPInt().getZExtValue();
-    } else {
-      assert(Memory);
-      std::memcpy(Memory, F.bitcastToAPInt().getRawData(),
-                  numWords() * sizeof(uint64_t));
-    }
-  }
-
-  void take(uint64_t *NewMemory) {
-    if (singleWord())
-      return;
-
-    if (Memory)
-      std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
-    Memory = NewMemory;
-  }
+  unsigned bitWidth() const { return F.semanticsSizeInBits(F.getSemantics()); }
 
   bool isSigned() const { return true; }
-  bool isNegative() const { return getValue().isNegative(); }
-  bool isZero() const { return getValue().isZero(); }
-  bool isNonZero() const { return getValue().isNonZero(); }
-  bool isMin() const { return getValue().isSmallest(); }
-  bool isMinusOne() const { return getValue().isExactlyValue(-1.0); }
-  bool isNan() const { return getValue().isNaN(); }
-  bool isSignaling() const { return getValue().isSignaling(); }
-  bool isInf() const { return getValue().isInfinity(); }
-  bool isFinite() const { return getValue().isFinite(); }
-  bool isNormal() const { return getValue().isNormal(); }
-  bool isDenormal() const { return getValue().isDenormal(); }
-  llvm::FPClassTest classify() const { return getValue().classify(); }
-  APFloat::fltCategory getCategory() const { return getValue().getCategory(); }
+  bool isNegative() const { return F.isNegative(); }
+  bool isZero() const { return F.isZero(); }
+  bool isNonZero() const { return F.isNonZero(); }
+  bool isMin() const { return F.isSmallest(); }
+  bool isMinusOne() const { return F.isExactlyValue(-1.0); }
+  bool isNan() const { return F.isNaN(); }
+  bool isSignaling() const { return F.isSignaling(); }
+  bool isInf() const { return F.isInfinity(); }
+  bool isFinite() const { return F.isFinite(); }
+  bool isNormal() const { return F.isNormal(); }
+  bool isDenormal() const { return F.isDenormal(); }
+  llvm::FPClassTest classify() const { return F.classify(); }
+  APFloat::fltCategory getCategory() const { return F.getCategory(); }
 
   ComparisonCategoryResult compare(const Floating &RHS) const {
-    llvm::APFloatBase::cmpResult CmpRes = getValue().compare(RHS.getValue());
+    llvm::APFloatBase::cmpResult CmpRes = F.compare(RHS.F);
     switch (CmpRes) {
     case llvm::APFloatBase::cmpLessThan:
       return ComparisonCategoryResult::Less;
@@ -171,130 +118,97 @@ public:
   static APFloat::opStatus fromIntegral(APSInt Val,
                                         const llvm::fltSemantics &Sem,
                                         llvm::RoundingMode RM,
-                                        Floating *Result) {
+                                        Floating &Result) {
     APFloat F = APFloat(Sem);
     APFloat::opStatus Status = F.convertFromAPInt(Val, Val.isSigned(), RM);
-    Result->copy(F);
+    Result = Floating(F);
     return Status;
   }
 
-  static void bitcastFromMemory(const std::byte *Buff,
-                                const llvm::fltSemantics &Sem,
-                                Floating *Result) {
+  static Floating bitcastFromMemory(const std::byte *Buff,
+                                    const llvm::fltSemantics &Sem) {
     size_t Size = APFloat::semanticsSizeInBits(Sem);
     llvm::APInt API(Size, true);
     llvm::LoadIntFromMemory(API, (const uint8_t *)Buff, Size / 8);
-    Result->copy(APFloat(Sem, API));
+
+    return Floating(APFloat(Sem, API));
   }
 
   void bitcastToMemory(std::byte *Buff) const {
-    llvm::APInt API = getValue().bitcastToAPInt();
+    llvm::APInt API = F.bitcastToAPInt();
     llvm::StoreIntToMemory(API, (uint8_t *)Buff, bitWidth() / 8);
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    return sizeof(Semantics) + (numWords() * sizeof(uint64_t));
+    return sizeof(llvm::fltSemantics *) +
+           (APFloat::semanticsSizeInBits(F.getSemantics()) / 8);
   }
 
   void serialize(std::byte *Buff) const {
-    std::memcpy(Buff, &Semantics, sizeof(Semantics));
-    if (singleWord()) {
-      std::memcpy(Buff + sizeof(Semantics), &Val, sizeof(uint64_t));
-    } else {
-      std::memcpy(Buff + sizeof(Semantics), Memory,
-                  numWords() * sizeof(uint64_t));
-    }
+    // Semantics followed by an APInt.
+    *reinterpret_cast<const llvm::fltSemantics **>(Buff) = &F.getSemantics();
+
+    llvm::APInt API = F.bitcastToAPInt();
+    llvm::StoreIntToMemory(API, (uint8_t *)(Buff + sizeof(void *)),
+                           bitWidth() / 8);
   }
 
-  static llvm::APFloatBase::Semantics
-  deserializeSemantics(const std::byte *Buff) {
-    return *reinterpret_cast<const llvm::APFloatBase::Semantics *>(Buff);
+  static Floating deserialize(const std::byte *Buff) {
+    const llvm::fltSemantics *Sem;
+    std::memcpy((void *)&Sem, Buff, sizeof(void *));
+    return bitcastFromMemory(Buff + sizeof(void *), *Sem);
   }
 
-  static void deserialize(const std::byte *Buff, Floating *Result) {
-    llvm::APFloatBase::Semantics Semantics;
-    std::memcpy(&Semantics, Buff, sizeof(Semantics));
-
-    unsigned BitWidth = llvm::APFloat::semanticsSizeInBits(
-        llvm::APFloatBase::EnumToSemantics(Semantics));
-    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
-
-    Result->Semantics = Semantics;
-    if (NumWords == 1 && !ALLOCATE_ALL) {
-      std::memcpy(&Result->Val, Buff + sizeof(Semantics), sizeof(uint64_t));
-    } else {
-      assert(Result->Memory);
-      std::memcpy(Result->Memory, Buff + sizeof(Semantics),
-                  NumWords * sizeof(uint64_t));
-    }
+  static Floating abs(const Floating &F) {
+    APFloat V = F.F;
+    if (V.isNegative())
+      V.changeSign();
+    return Floating(V);
   }
 
   // -------
 
   static APFloat::opStatus add(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    APFloat LHS = A.getValue();
-    APFloat RHS = B.getValue();
-
-    auto Status = LHS.add(RHS, RM);
-    R->copy(LHS);
-    return Status;
+    *R = Floating(A.F);
+    return R->F.add(B.F, RM);
   }
 
   static APFloat::opStatus increment(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.getSemantics(), 1);
-    APFloat LHS = A.getValue();
-
-    auto Status = LHS.add(One, RM);
-    R->copy(LHS);
-    return Status;
+    APFloat One(A.F.getSemantics(), 1);
+    *R = Floating(A.F);
+    return R->F.add(One, RM);
   }
 
   static APFloat::opStatus sub(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    APFloat LHS = A.getValue();
-    APFloat RHS = B.getValue();
-
-    auto Status = LHS.subtract(RHS, RM);
-    R->copy(LHS);
-    return Status;
+    *R = Floating(A.F);
+    return R->F.subtract(B.F, RM);
   }
 
   static APFloat::opStatus decrement(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.getSemantics(), 1);
-    APFloat LHS = A.getValue();
-
-    auto Status = LHS.subtract(One, RM);
-    R->copy(LHS);
-    return Status;
+    APFloat One(A.F.getSemantics(), 1);
+    *R = Floating(A.F);
+    return R->F.subtract(One, RM);
   }
 
   static APFloat::opStatus mul(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-
-    APFloat LHS = A.getValue();
-    APFloat RHS = B.getValue();
-
-    auto Status = LHS.multiply(RHS, RM);
-    R->copy(LHS);
-    return Status;
+    *R = Floating(A.F);
+    return R->F.multiply(B.F, RM);
   }
 
   static APFloat::opStatus div(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    APFloat LHS = A.getValue();
-    APFloat RHS = B.getValue();
-
-    auto Status = LHS.divide(RHS, RM);
-    R->copy(LHS);
-    return Status;
+    *R = Floating(A.F);
+    return R->F.divide(B.F, RM);
   }
 
   static bool neg(const Floating &A, Floating *R) {
-    R->copy(-A.getValue());
+    *R = -A;
     return false;
   }
 };
diff --git a/clang/lib/AST/ByteCode/Integral.h b/clang/lib/AST/ByteCode/Integral.h
index af5cd2d13ecc..13fdb5369f2b 100644
--- a/clang/lib/AST/ByteCode/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -99,9 +99,6 @@ public:
   bool operator>=(Integral RHS) const { return V >= RHS.V; }
   bool operator==(Integral RHS) const { return V == RHS.V; }
   bool operator!=(Integral RHS) const { return V != RHS.V; }
-  bool operator>=(unsigned RHS) const {
-    return static_cast<unsigned>(V) >= RHS;
-  }
 
   bool operator>(unsigned RHS) const {
     return V >= 0 && static_cast<unsigned>(V) > RHS;
diff --git a/clang/lib/AST/ByteCode/IntegralAP.h b/clang/lib/AST/ByteCode/IntegralAP.h
index 61cbd14ad174..8ee08dfb5cfe 100644
--- a/clang/lib/AST/ByteCode/IntegralAP.h
+++ b/clang/lib/AST/ByteCode/IntegralAP.h
@@ -28,19 +28,12 @@ namespace interp {
 
 using APInt = llvm::APInt;
 using APSInt = llvm::APSInt;
+template <unsigned Bits, bool Signed> class Integral;
 
-/// If an IntegralAP is constructed from Memory, it DOES NOT OWN THAT MEMORY.
-/// It will NOT copy the memory (unless, of course, copy() is called) and it
-/// won't alllocate anything. The allocation should happen via InterpState or
-/// Program.
 template <bool Signed> class IntegralAP final {
-public:
-  union {
-    uint64_t *Memory = nullptr;
-    uint64_t Val;
-  };
-  uint32_t BitWidth = 0;
+private:
   friend IntegralAP<!Signed>;
+  APInt V;
 
   template <typename T, bool InputSigned>
   static T truncateCast(const APInt &V) {
@@ -59,86 +52,52 @@ public:
                                : V.trunc(BitSize).getZExtValue();
   }
 
-  APInt getValue() const {
-    if (singleWord())
-      return APInt(BitWidth, Val, Signed);
-    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
-    return llvm::APInt(BitWidth, NumWords, Memory);
-  }
-
 public:
   using AsUnsigned = IntegralAP<false>;
 
-  void take(uint64_t *NewMemory) {
-    assert(!singleWord());
-    std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
-    Memory = NewMemory;
-  }
+  template <typename T>
+  IntegralAP(T Value, unsigned BitWidth)
+      : V(APInt(BitWidth, static_cast<uint64_t>(Value), Signed)) {}
 
-  void copy(const APInt &V) {
-    assert(BitWidth == V.getBitWidth());
-    assert(numWords() == V.getNumWords());
+  IntegralAP(APInt V) : V(V) {}
+  /// Arbitrary value for uninitialized variables.
+  IntegralAP() : IntegralAP(Signed ? -1 : 7, 3) {}
 
-    if (V.isSingleWord()) {
-      if constexpr (Signed)
-        Val = V.getSExtValue();
-      else
-        Val = V.getZExtValue();
-      return;
-    }
-    assert(Memory);
-    std::memcpy(Memory, V.getRawData(), V.getNumWords() * sizeof(uint64_t));
-  }
-
-  // Constructors.
-  IntegralAP() = default;
-  IntegralAP(unsigned BitWidth) : BitWidth(BitWidth) {}
-  IntegralAP(uint64_t *Memory, unsigned BitWidth)
-      : Memory(Memory), BitWidth(BitWidth) {}
-  IntegralAP(const APInt &V) : BitWidth(V.getBitWidth()) {
-    if (V.isSingleWord()) {
-      Val = Signed ? V.getSExtValue() : V.getZExtValue();
-    } else {
-      Memory = const_cast<uint64_t *>(V.getRawData());
-    }
-  }
-
-  IntegralAP operator-() const { return IntegralAP(-getValue()); }
+  IntegralAP operator-() const { return IntegralAP(-V); }
   IntegralAP operator-(const IntegralAP &Other) const {
-    return IntegralAP(getValue() - Other.getValue());
+    return IntegralAP(V - Other.V);
   }
   bool operator>(const IntegralAP &RHS) const {
     if constexpr (Signed)
-      return getValue().sgt(RHS.getValue());
-    return getValue().ugt(RHS.getValue());
+      return V.ugt(RHS.V);
+    return V.sgt(RHS.V);
   }
-  bool operator>=(unsigned RHS) const {
+  bool operator>=(IntegralAP RHS) const {
     if constexpr (Signed)
-      return getValue().sge(RHS);
-    return getValue().uge(RHS);
+      return V.uge(RHS.V);
+    return V.sge(RHS.V);
   }
   bool operator<(IntegralAP RHS) const {
     if constexpr (Signed)
-      return getValue().slt(RHS.getValue());
-    return getValue().ult(RHS.getValue());
+      return V.slt(RHS.V);
+    return V.slt(RHS.V);
+  }
+  bool operator<=(IntegralAP RHS) const {
+    if constexpr (Signed)
+      return V.ult(RHS.V);
+    return V.ult(RHS.V);
   }
 
   template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
   explicit operator Ty() const {
-    return truncateCast<Ty, Signed>(getValue());
+    return truncateCast<Ty, Signed>(V);
   }
 
   template <typename T> static IntegralAP from(T Value, unsigned NumBits = 0) {
-    if (NumBits == 0)
-      NumBits = sizeof(T) * 8;
     assert(NumBits > 0);
     APInt Copy = APInt(NumBits, static_cast<uint64_t>(Value), Signed);
-    assert(false);
-    return IntegralAP<Signed>(Copy);
-  }
 
-  static IntegralAP from(const APInt &Value) {
-    return IntegralAP<Signed>(Value);
+    return IntegralAP<Signed>(Copy);
   }
 
   template <bool InputSigned>
@@ -147,45 +106,52 @@ public:
       NumBits = V.bitWidth();
 
     if constexpr (InputSigned)
-      return IntegralAP<Signed>(V.getValue().sextOrTrunc(NumBits));
-    return IntegralAP<Signed>(V.getValue().zextOrTrunc(NumBits));
+      return IntegralAP<Signed>(V.V.sextOrTrunc(NumBits));
+    return IntegralAP<Signed>(V.V.zextOrTrunc(NumBits));
+  }
+
+  template <unsigned Bits, bool InputSigned>
+  static IntegralAP from(Integral<Bits, InputSigned> I, unsigned BitWidth) {
+    return IntegralAP<Signed>(I.toAPInt(BitWidth));
+  }
+
+  static IntegralAP zero(int32_t BitWidth) {
+    APInt V = APInt(BitWidth, 0LL, Signed);
+    return IntegralAP(V);
   }
 
-  constexpr unsigned bitWidth() const { return BitWidth; }
-  constexpr unsigned numWords() const { return APInt::getNumWords(BitWidth); }
-  constexpr bool singleWord() const { return numWords() == 1; }
+  constexpr unsigned bitWidth() const { return V.getBitWidth(); }
 
   APSInt toAPSInt(unsigned Bits = 0) const {
     if (Bits == 0)
       Bits = bitWidth();
 
-    APInt V = getValue();
     if constexpr (Signed)
-      return APSInt(getValue().sext(Bits), !Signed);
+      return APSInt(V.sext(Bits), !Signed);
     else
-      return APSInt(getValue().zext(Bits), !Signed);
+      return APSInt(V.zext(Bits), !Signed);
   }
   APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
-  bool isZero() const { return getValue().isZero(); }
+  bool isZero() const { return V.isZero(); }
   bool isPositive() const {
     if constexpr (Signed)
-      return getValue().isNonNegative();
+      return V.isNonNegative();
     return true;
   }
   bool isNegative() const {
     if constexpr (Signed)
-      return !getValue().isNonNegative();
+      return !V.isNonNegative();
     return false;
   }
-  bool isMin() const { return getValue().isMinValue(); }
-  bool isMax() const { return getValue().isMaxValue(); }
+  bool isMin() const { return V.isMinValue(); }
+  bool isMax() const { return V.isMaxValue(); }
   static constexpr bool isSigned() { return Signed; }
-  bool isMinusOne() const { return Signed && getValue().isAllOnes(); }
+  bool isMinusOne() const { return Signed && V == -1; }
 
-  unsigned countLeadingZeros() const { return getValue().countl_zero(); }
+  unsigned countLeadingZeros() const { return V.countl_zero(); }
 
-  void print(llvm::raw_ostream &OS) const { getValue().print(OS, Signed); }
+  void print(llvm::raw_ostream &OS) const { V.print(OS, Signed);}
   std::string toDiagnosticString(const ASTContext &Ctx) const {
     std::string NameStr;
     llvm::raw_string_ostream OS(NameStr);
@@ -195,57 +161,53 @@ public:
 
   IntegralAP truncate(unsigned BitWidth) const {
     if constexpr (Signed)
-      return IntegralAP(
-          getValue().trunc(BitWidth).sextOrTrunc(this->bitWidth()));
+      return IntegralAP(V.trunc(BitWidth).sextOrTrunc(this->bitWidth()));
     else
-      return IntegralAP(
-          getValue().trunc(BitWidth).zextOrTrunc(this->bitWidth()));
+      return IntegralAP(V.trunc(BitWidth).zextOrTrunc(this->bitWidth()));
   }
 
   IntegralAP<false> toUnsigned() const {
-    return IntegralAP<false>(Memory, BitWidth);
+    APInt Copy = V;
+    return IntegralAP<false>(Copy);
   }
 
   void bitcastToMemory(std::byte *Dest) const {
-    llvm::StoreIntToMemory(getValue(), (uint8_t *)Dest, bitWidth() / 8);
+    llvm::StoreIntToMemory(V, (uint8_t *)Dest, bitWidth() / 8);
   }
 
-  static void bitcastFromMemory(const std::byte *Src, unsigned BitWidth,
-                                IntegralAP *Result) {
+  static IntegralAP bitcastFromMemory(const std::byte *Src, unsigned BitWidth) {
     APInt V(BitWidth, static_cast<uint64_t>(0), Signed);
     llvm::LoadIntFromMemory(V, (const uint8_t *)Src, BitWidth / 8);
-    Result->copy(V);
+    return IntegralAP(V);
   }
 
   ComparisonCategoryResult compare(const IntegralAP &RHS) const {
     assert(Signed == RHS.isSigned());
     assert(bitWidth() == RHS.bitWidth());
-    APInt V1 = getValue();
-    APInt V2 = RHS.getValue();
     if constexpr (Signed) {
-      if (V1.slt(V2))
+      if (V.slt(RHS.V))
         return ComparisonCategoryResult::Less;
-      if (V1.sgt(V2))
+      if (V.sgt(RHS.V))
         return ComparisonCategoryResult::Greater;
       return ComparisonCategoryResult::Equal;
     }
 
     assert(!Signed);
-    if (V1.ult(V2))
+    if (V.ult(RHS.V))
       return ComparisonCategoryResult::Less;
-    if (V1.ugt(V2))
+    if (V.ugt(RHS.V))
       return ComparisonCategoryResult::Greater;
     return ComparisonCategoryResult::Equal;
   }
 
   static bool increment(IntegralAP A, IntegralAP *R) {
-    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
-    return add(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
+    IntegralAP<Signed> One(1, A.bitWidth());
+    return add(A, One, A.bitWidth() + 1, R);
   }
 
   static bool decrement(IntegralAP A, IntegralAP *R) {
-    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
-    return sub(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
+    IntegralAP<Signed> One(1, A.bitWidth());
+    return sub(A, One, A.bitWidth() + 1, R);
   }
 
   static bool add(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
@@ -262,97 +224,87 @@ public:
 
   static bool rem(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      R->copy(A.getValue().srem(B.getValue()));
+      *R = IntegralAP(A.V.srem(B.V));
     else
-      R->copy(A.getValue().urem(B.getValue()));
+      *R = IntegralAP(A.V.urem(B.V));
     return false;
   }
 
   static bool div(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      R->copy(A.getValue().sdiv(B.getValue()));
+      *R = IntegralAP(A.V.sdiv(B.V));
     else
-      R->copy(A.getValue().udiv(B.getValue()));
+      *R = IntegralAP(A.V.udiv(B.V));
     return false;
   }
 
   static bool bitAnd(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    R->copy(A.getValue() & B.getValue());
+    *R = IntegralAP(A.V & B.V);
     return false;
   }
 
   static bool bitOr(IntegralAP A, IntegralAP B, unsigned OpBits,
                     IntegralAP *R) {
-    R->copy(A.getValue() | B.getValue());
+    *R = IntegralAP(A.V | B.V);
     return false;
   }
 
   static bool bitXor(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    R->copy(A.getValue() ^ B.getValue());
+    *R = IntegralAP(A.V ^ B.V);
     return false;
   }
 
   static bool neg(const IntegralAP &A, IntegralAP *R) {
-    APInt AI = A.getValue();
+    APInt AI = A.V;
     AI.negate();
-    R->copy(AI);
+    *R = IntegralAP(AI);
     return false;
   }
 
   static bool comp(IntegralAP A, IntegralAP *R) {
-    R->copy(~A.getValue());
+    *R = IntegralAP(~A.V);
     return false;
   }
 
   static void shiftLeft(const IntegralAP A, const IntegralAP B, unsigned OpBits,
                         IntegralAP *R) {
-    *R = IntegralAP(A.getValue().shl(B.getValue().getZExtValue()));
+    *R = IntegralAP(A.V.shl(B.V.getZExtValue()));
   }
 
   static void shiftRight(const IntegralAP A, const IntegralAP B,
                          unsigned OpBits, IntegralAP *R) {
-    unsigned ShiftAmount = B.getValue().getZExtValue();
+    unsigned ShiftAmount = B.V.getZExtValue();
     if constexpr (Signed)
-      R->copy(A.getValue().ashr(ShiftAmount));
+      *R = IntegralAP(A.V.ashr(ShiftAmount));
     else
-      R->copy(A.getValue().lshr(ShiftAmount));
+      *R = IntegralAP(A.V.lshr(ShiftAmount));
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    assert(BitWidth != 0);
-    uint64_t NumWords = llvm::APInt::getNumWords(bitWidth());
-    return sizeof(uint32_t) + (NumWords * sizeof(uint64_t));
+    // 4 bytes for the BitWidth followed by N bytes for the actual APInt.
+    return sizeof(uint32_t) + (V.getBitWidth() / CHAR_BIT);
   }
 
   void serialize(std::byte *Buff) const {
-    std::memcpy(Buff, &BitWidth, sizeof(uint32_t));
-    if (singleWord())
-      std::memcpy(Buff + sizeof(uint32_t), &Val, sizeof(uint64_t));
-    else {
-      uint64_t NumWords = llvm::APInt::getNumWords(bitWidth());
-      std::memcpy(Buff + sizeof(uint32_t), Memory, NumWords * sizeof(uint64_t));
-    }
-  }
+    assert(V.getBitWidth() < std::numeric_limits<uint8_t>::max());
+    uint32_t BitWidth = V.getBitWidth();
 
-  static uint64_t deserializeSize(const std::byte *Buff) {
-    return *reinterpret_cast<const uint32_t *>(Buff);
+    std::memcpy(Buff, &BitWidth, sizeof(uint32_t));
+    llvm::StoreIntToMemory(V, (uint8_t *)(Buff + sizeof(uint32_t)),
+                           BitWidth / CHAR_BIT);
   }
 
-  static void deserialize(const std::byte *Buff, IntegralAP<Signed> *Result) {
-    uint64_t BitWidth = Result->BitWidth;
-    uint64_t NumWords = llvm::APInt::getNumWords(BitWidth);
-    assert(BitWidth == Result->BitWidth);
+  static IntegralAP<Signed> deserialize(const std::byte *Buff) {
+    uint32_t BitWidth;
+    std::memcpy(&BitWidth, Buff, sizeof(uint32_t));
+    IntegralAP<Signed> Val(APInt(BitWidth, 0ull, !Signed));
 
-    if (NumWords == 1)
-      std::memcpy(&Result->Val, Buff + sizeof(uint32_t), sizeof(uint64_t));
-    else {
-      assert(Result->Memory);
-      std::memcpy(Result->Memory, Buff + sizeof(uint32_t),
-                  NumWords * sizeof(uint64_t));
-    }
+    llvm::LoadIntFromMemory(Val.V, (const uint8_t *)Buff + sizeof(uint32_t),
+                            BitWidth / CHAR_BIT);
+    return Val;
   }
 
 private:
@@ -360,7 +312,7 @@ private:
   static bool CheckAddSubMulUB(const IntegralAP &A, const IntegralAP &B,
                                unsigned BitWidth, IntegralAP *R) {
     if constexpr (!Signed) {
-      R->copy(Op<APInt>{}(A.getValue(), B.getValue()));
+      R->V = Op<APInt>{}(A.V, B.V);
       return false;
     }
 
@@ -368,7 +320,7 @@ private:
     const APSInt &RHS = B.toAPSInt();
     APSInt Value = Op<APSInt>{}(LHS.extend(BitWidth), RHS.extend(BitWidth));
     APSInt Result = Value.trunc(LHS.getBitWidth());
-    R->copy(Result);
+    R->V = Result;
 
     return Result.extend(BitWidth) != Value;
   }
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 1e2032feabb6..5c8abffb3a99 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1935,10 +1935,8 @@ bool CastPointerIntegralAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
-  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
-
-  S.Stk.push<IntegralAP<false>>(Result);
+  S.Stk.push<IntegralAP<false>>(
+      IntegralAP<false>::from(Ptr.getIntegerRepresentation(), BitWidth));
   return true;
 }
 
@@ -1948,10 +1946,8 @@ bool CastPointerIntegralAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
-  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
-
-  S.Stk.push<IntegralAP<true>>(Result);
+  S.Stk.push<IntegralAP<true>>(
+      IntegralAP<true>::from(Ptr.getIntegerRepresentation(), BitWidth));
   return true;
 }
 
@@ -2057,100 +2053,6 @@ bool arePotentiallyOverlappingStringLiterals(const Pointer &LHS,
   return Shorter == Longer.take_front(Shorter.size());
 }
 
-static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr,
-                                PrimType T) {
-
-  if (T == PT_IntAPS) {
-    auto &Val = Ptr.deref<IntegralAP<true>>();
-    if (!Val.singleWord()) {
-      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-      Val.take(NewMemory);
-    }
-  } else if (T == PT_IntAP) {
-    auto &Val = Ptr.deref<IntegralAP<false>>();
-    if (!Val.singleWord()) {
-      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-      Val.take(NewMemory);
-    }
-  } else if (T == PT_Float) {
-    auto &Val = Ptr.deref<Floating>();
-    if (!Val.singleWord()) {
-      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-      Val.take(NewMemory);
-    }
-  }
-}
-
-template <typename T>
-static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr) {
-  assert(needsAlloc<T>());
-  auto &Val = Ptr.deref<T>();
-  if (!Val.singleWord()) {
-    uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-    Val.take(NewMemory);
-  }
-}
-
-static void finishGlobalRecurse(InterpState &S, const Pointer &Ptr) {
-  if (const Record *R = Ptr.getRecord()) {
-    for (const Record::Field &Fi : R->fields()) {
-      if (Fi.Desc->isPrimitive()) {
-        TYPE_SWITCH_ALLOC(Fi.Desc->getPrimType(), {
-          copyPrimitiveMemory<T>(S, Ptr.atField(Fi.Offset));
-        });
-        copyPrimitiveMemory(S, Ptr.atField(Fi.Offset), Fi.Desc->getPrimType());
-      } else
-        finishGlobalRecurse(S, Ptr.atField(Fi.Offset));
-    }
-    return;
-  }
-
-  if (const Descriptor *D = Ptr.getFieldDesc(); D && D->isArray()) {
-    unsigned NumElems = D->getNumElems();
-    if (NumElems == 0)
-      return;
-
-    if (D->isPrimitiveArray()) {
-      PrimType PT = D->getPrimType();
-      if (!needsAlloc(PT))
-        return;
-      assert(NumElems >= 1);
-      const Pointer EP = Ptr.atIndex(0);
-      bool AllSingleWord = true;
-      TYPE_SWITCH_ALLOC(PT, {
-        if (!EP.deref<T>().singleWord()) {
-          copyPrimitiveMemory<T>(S, EP);
-          AllSingleWord = false;
-        }
-      });
-      if (AllSingleWord)
-        return;
-      for (unsigned I = 1; I != D->getNumElems(); ++I) {
-        const Pointer EP = Ptr.atIndex(I);
-        copyPrimitiveMemory(S, EP, PT);
-      }
-    } else {
-      assert(D->isCompositeArray());
-      for (unsigned I = 0; I != D->getNumElems(); ++I) {
-        const Pointer EP = Ptr.atIndex(I).narrow();
-        finishGlobalRecurse(S, EP);
-      }
-    }
-  }
-}
-
-bool FinishInitGlobal(InterpState &S, CodePtr OpPC) {
-  const Pointer &Ptr = S.Stk.pop<Pointer>();
-
-  finishGlobalRecurse(S, Ptr);
-  if (Ptr.canBeInitialized()) {
-    Ptr.initialize();
-    Ptr.activate();
-  }
-
-  return true;
-}
-
 // https://github.com/llvm/llvm-project/issues/102513
 #if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
 #pragma optimize("", off)
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 66d3e6d79e8b..ae3d4a441a79 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -189,7 +189,7 @@ bool CheckShift(InterpState &S, CodePtr OpPC, const LT &LHS, const RT &RHS,
 
   // C++11 [expr.shift]p1: Shift width must be less than the bit width of
   // the shifted type.
-  if (Bits > 1 && RHS >= Bits) {
+  if (Bits > 1 && RHS >= RT::from(Bits, RHS.bitWidth())) {
     const Expr *E = S.Current->getExpr(OpPC);
     const APSInt Val = RHS.toAPSInt();
     QualType Ty = E->getType();
@@ -370,9 +370,6 @@ bool AddSubMulHelper(InterpState &S, CodePtr OpPC, unsigned Bits, const T &LHS,
                      const T &RHS) {
   // Fast path - add the numbers with fixed width.
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(LHS.bitWidth());
-
   if (!OpFW(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -411,7 +408,6 @@ bool Add(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
-
   return AddSubMulHelper<T, T::add, std::plus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -427,7 +423,7 @@ inline bool Addf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result = S.allocFloat(LHS.getSemantics());
+  Floating Result;
   auto Status = Floating::add(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -438,7 +434,6 @@ bool Sub(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
-
   return AddSubMulHelper<T, T::sub, std::minus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -447,7 +442,7 @@ inline bool Subf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result = S.allocFloat(LHS.getSemantics());
+  Floating Result;
   auto Status = Floating::sub(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -458,7 +453,6 @@ bool Mul(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() * 2;
-
   return AddSubMulHelper<T, T::mul, std::multiplies>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -467,10 +461,8 @@ inline bool Mulf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result = S.allocFloat(LHS.getSemantics());
-
+  Floating Result;
   auto Status = Floating::mul(LHS, RHS, getRoundingMode(FPO), &Result);
-
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -492,14 +484,9 @@ inline bool Mulc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexMul(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    Floating RA = S.allocFloat(A.getSemantics());
-    RA.copy(ResR);
-    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
+    Result.atIndex(0).deref<Floating>() = Floating(ResR);
     Result.atIndex(0).initialize();
-
-    Floating RI = S.allocFloat(A.getSemantics());
-    RI.copy(ResI);
-    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
+    Result.atIndex(1).deref<Floating>() = Floating(ResI);
     Result.atIndex(1).initialize();
     Result.initialize();
   } else {
@@ -552,20 +539,10 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    // Result.atIndex(0).deref<Floating>() = Floating(ResR);
-    // Result.atIndex(0).initialize();
-    // Result.atIndex(1).deref<Floating>() = Floating(ResI);
-    // Result.atIndex(1).initialize();
-
-    Floating RA = S.allocFloat(A.getSemantics());
-    RA.copy(ResR);
-    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
+    Result.atIndex(0).deref<Floating>() = Floating(ResR);
     Result.atIndex(0).initialize();
-
-    Floating RI = S.allocFloat(A.getSemantics());
-    RI.copy(ResI);
-    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
-
+    Result.atIndex(1).deref<Floating>() = Floating(ResI);
+    Result.atIndex(1).initialize();
     Result.initialize();
   } else {
     // Integer element type.
@@ -631,12 +608,9 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitAnd(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-  unsigned Bits = RHS.bitWidth();
 
+  unsigned Bits = RHS.bitWidth();
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(Bits);
-
   if (!T::bitAnd(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -651,12 +625,9 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitOr(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-  unsigned Bits = RHS.bitWidth();
 
+  unsigned Bits = RHS.bitWidth();
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(Bits);
-
   if (!T::bitOr(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -673,11 +644,7 @@ bool BitXor(InterpState &S, CodePtr OpPC) {
   const T &LHS = S.Stk.pop<T>();
 
   unsigned Bits = RHS.bitWidth();
-
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(Bits);
-
   if (!T::bitXor(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -692,15 +659,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Rem(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
+  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(LHS.bitWidth());
-
   if (!T::rem(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -715,15 +679,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Div(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
+  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(LHS.bitWidth());
-
   if (!T::div(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -746,10 +707,8 @@ inline bool Divf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
     return false;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-
-  Floating Result = S.allocFloat(LHS.getSemantics());
+  Floating Result;
   auto Status = Floating::div(LHS, RHS, getRoundingMode(FPO), &Result);
-
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -771,44 +730,31 @@ inline bool Inv(InterpState &S, CodePtr OpPC) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Neg(InterpState &S, CodePtr OpPC) {
   const T &Value = S.Stk.pop<T>();
+  T Result;
 
-  if constexpr (std::is_same_v<T, Floating>) {
-    T Result = S.allocFloat(Value.getSemantics());
-
-    if (!T::neg(Value, &Result)) {
-      S.Stk.push<T>(Result);
-      return true;
-    }
-    return false;
-  } else {
-    T Result;
-    if constexpr (needsAlloc<T>())
-      Result = S.allocAP<T>(Value.bitWidth());
-
-    if (!T::neg(Value, &Result)) {
-      S.Stk.push<T>(Result);
-      return true;
-    }
-
-    assert(isIntegralType(Name) &&
-           "don't expect other types to fail at constexpr negation");
+  if (!T::neg(Value, &Result)) {
     S.Stk.push<T>(Result);
+    return true;
+  }
 
-    APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
-    if (S.checkingForUndefinedBehavior()) {
-      const Expr *E = S.Current->getExpr(OpPC);
-      QualType Type = E->getType();
-      SmallString<32> Trunc;
-      NegatedValue.trunc(Result.bitWidth())
-          .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
-                    /*UpperCase=*/true, /*InsertSeparators=*/true);
-      S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
-          << Trunc << Type << E->getSourceRange();
-      return true;
-    }
+  assert(isIntegralType(Name) &&
+         "don't expect other types to fail at constexpr negation");
+  S.Stk.push<T>(Result);
 
-    return handleOverflow(S, OpPC, NegatedValue);
+  APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
+  if (S.checkingForUndefinedBehavior()) {
+    const Expr *E = S.Current->getExpr(OpPC);
+    QualType Type = E->getType();
+    SmallString<32> Trunc;
+    NegatedValue.trunc(Result.bitWidth())
+        .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
+                  /*UpperCase=*/true, /*InsertSeparators=*/true);
+    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
+        << Trunc << Type << E->getSourceRange();
+    return true;
   }
+
+  return handleOverflow(S, OpPC, NegatedValue);
 }
 
 enum class PushVal : bool {
@@ -837,8 +783,6 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
   const T &Value = Ptr.deref<T>();
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(Value.bitWidth());
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<T>(Value);
@@ -946,6 +890,7 @@ bool PreDec(InterpState &S, CodePtr OpPC, bool CanOverflow) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckLoad(S, OpPC, Ptr, AK_Decrement))
     return false;
+
   return IncDecHelper<T, IncDecOp::Dec, PushVal::No>(S, OpPC, Ptr, CanOverflow);
 }
 
@@ -953,7 +898,7 @@ template <IncDecOp Op, PushVal DoPush>
 bool IncDecFloatHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                        uint32_t FPOI) {
   Floating Value = Ptr.deref<Floating>();
-  Floating Result = S.allocFloat(Value.getSemantics());
+  Floating Result;
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<Floating>(Value);
@@ -1007,15 +952,12 @@ inline bool DecfPop(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Comp(InterpState &S, CodePtr OpPC) {
   const T &Val = S.Stk.pop<T>();
-
   T Result;
-  if constexpr (needsAlloc<T>())
-    Result = S.allocAP<T>(Val.bitWidth());
-
   if (!T::comp(Val, &Result)) {
     S.Stk.push<T>(Result);
     return true;
   }
+
   return false;
 }
 
@@ -1383,23 +1325,10 @@ bool Flip(InterpState &S, CodePtr OpPC) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Const(InterpState &S, CodePtr OpPC, const T &Arg) {
-  if constexpr (needsAlloc<T>()) {
-    T Result = S.allocAP<T>(Arg.bitWidth());
-    Result.copy(Arg.toAPSInt());
-    S.Stk.push<T>(Result);
-    return true;
-  }
   S.Stk.push<T>(Arg);
   return true;
 }
 
-inline bool ConstFloat(InterpState &S, CodePtr OpPC, const Floating &F) {
-  Floating Result = S.allocFloat(F.getSemantics());
-  Result.copy(F.getAPFloat());
-  S.Stk.push<Floating>(Result);
-  return true;
-}
-
 //===----------------------------------------------------------------------===//
 // Get/Set Local/Param/Global/This
 //===----------------------------------------------------------------------===//
@@ -1554,24 +1483,7 @@ bool SetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool InitGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &P = S.P.getGlobal(I);
-
   P.deref<T>() = S.Stk.pop<T>();
-
-  if constexpr (std::is_same_v<T, Floating>) {
-    auto &Val = P.deref<Floating>();
-    if (!Val.singleWord()) {
-      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-      Val.take(NewMemory);
-    }
-
-  } else if constexpr (needsAlloc<T>()) {
-    auto &Val = P.deref<T>();
-    if (!Val.singleWord()) {
-      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
-      Val.take(NewMemory);
-    }
-  }
-
   P.initialize();
   return true;
 }
@@ -1673,22 +1585,7 @@ bool InitBitField(InterpState &S, CodePtr OpPC, const Record::Field *F) {
   assert(F->isBitField());
   const T &Value = S.Stk.pop<T>();
   const Pointer &Field = S.Stk.peek<Pointer>().atField(F->Offset);
-
-  if constexpr (needsAlloc<T>()) {
-    T Result = S.allocAP<T>(Value.bitWidth());
-    if (T::isSigned())
-      Result.copy(Value.toAPSInt()
-                      .trunc(F->Decl->getBitWidthValue())
-                      .sextOrTrunc(Value.bitWidth()));
-    else
-      Result.copy(Value.toAPSInt()
-                      .trunc(F->Decl->getBitWidthValue())
-                      .zextOrTrunc(Value.bitWidth()));
-
-    Field.deref<T>() = Result;
-  } else {
-    Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
-  }
+  Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
   Field.activate();
   Field.initialize();
   return true;
@@ -1868,8 +1765,6 @@ inline bool FinishInit(InterpState &S, CodePtr OpPC) {
   return true;
 }
 
-bool FinishInitGlobal(InterpState &S, CodePtr OpPC);
-
 inline bool Dump(InterpState &S, CodePtr OpPC) {
   S.Stk.dump();
   return true;
@@ -2376,8 +2271,7 @@ template <PrimType TIn, PrimType TOut> bool Cast(InterpState &S, CodePtr OpPC) {
 inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
                    llvm::RoundingMode RM) {
   Floating F = S.Stk.pop<Floating>();
-  Floating Result = S.allocFloat(*Sem);
-  F.toSemantics(Sem, RM, &Result);
+  Floating Result = F.toSemantics(Sem, RM);
   S.Stk.push<Floating>(Result);
   return true;
 }
@@ -2401,25 +2295,15 @@ inline bool CastFixedPoint(InterpState &S, CodePtr OpPC, uint32_t FPS) {
 /// to know what bitwidth the result should be.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
-  // Copy data.
-  {
-    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
-    Result.copy(Source);
-  }
-  S.Stk.push<IntegralAP<false>>(Result);
+  S.Stk.push<IntegralAP<false>>(
+      IntegralAP<false>::from(S.Stk.pop<T>(), BitWidth));
   return true;
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
-  // Copy data.
-  {
-    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
-    Result.copy(Source);
-  }
-  S.Stk.push<IntegralAP<true>>(Result);
+  S.Stk.push<IntegralAP<true>>(
+      IntegralAP<true>::from(S.Stk.pop<T>(), BitWidth));
   return true;
 }
 
@@ -2428,11 +2312,11 @@ bool CastIntegralFloating(InterpState &S, CodePtr OpPC,
                           const llvm::fltSemantics *Sem, uint32_t FPOI) {
   const T &From = S.Stk.pop<T>();
   APSInt FromAP = From.toAPSInt();
+  Floating Result;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result = S.allocFloat(*Sem);
   auto Status =
-      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), &Result);
+      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), Result);
   S.Stk.push<Floating>(Result);
 
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -2481,12 +2365,7 @@ static inline bool CastFloatingIntegralAP(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-
-  auto ResultAP = S.allocAP<IntegralAP<false>>(BitWidth);
-  ResultAP.copy(Result);
-
-  S.Stk.push<IntegralAP<false>>(ResultAP);
-
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2502,12 +2381,7 @@ static inline bool CastFloatingIntegralAPS(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-
-  auto ResultAP = S.allocAP<IntegralAP<true>>(BitWidth);
-  ResultAP.copy(Result);
-
-  S.Stk.push<IntegralAP<true>>(ResultAP);
-
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2567,9 +2441,8 @@ static inline bool CastFloatingFixedPoint(InterpState &S, CodePtr OpPC,
 static inline bool CastFixedPointFloating(InterpState &S, CodePtr OpPC,
                                           const llvm::fltSemantics *Sem) {
   const auto &Fixed = S.Stk.pop<FixedPoint>();
-  Floating Result = S.allocFloat(*Sem);
-  Result.copy(Fixed.toFloat(Sem));
-  S.Stk.push<Floating>(Result);
+
+  S.Stk.push<Floating>(Fixed.toFloat(Sem));
   return true;
 }
 
@@ -2633,18 +2506,12 @@ bool Zero(InterpState &S, CodePtr OpPC) {
 }
 
 static inline bool ZeroIntAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
-  if (!Result.singleWord())
-    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
-  S.Stk.push<IntegralAP<false>>(Result);
+  S.Stk.push<IntegralAP<false>>(IntegralAP<false>::zero(BitWidth));
   return true;
 }
 
 static inline bool ZeroIntAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
-  if (!Result.singleWord())
-    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
-  S.Stk.push<IntegralAP<true>>(Result);
+  S.Stk.push<IntegralAP<true>>(IntegralAP<true>::zero(BitWidth));
   return true;
 }
 
@@ -2711,9 +2578,7 @@ inline bool RVOPtr(InterpState &S, CodePtr OpPC) {
 //===----------------------------------------------------------------------===//
 
 template <class LT, class RT, ShiftDir Dir>
-inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
-                    LT *Result) {
-
+inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
   const unsigned Bits = LHS.bitWidth();
 
   // OpenCL 6.3j: shift values are effectively % word size of LHS.
@@ -2731,7 +2596,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
     RHS = -RHS;
     return DoShift<LT, RT,
                    Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
-        S, OpPC, LHS, RHS, Result);
+        S, OpPC, LHS, RHS);
   }
 
   if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
@@ -2779,7 +2644,6 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
       // Do the shift on potentially signed LT, then convert to unsigned type.
       LT A;
       LT::shiftRight(LHS, LT::from(RHS, Bits), Bits, &A);
-      // LT::shiftRight(LHS, LT(RHSTemp), Bits, &A);
       R = LT::AsUnsigned::from(A);
     }
   }
@@ -2788,48 +2652,6 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
   return true;
 }
 
-/// A version of DoShift that works on IntegralAP.
-template <class LT, class RT, ShiftDir Dir>
-inline bool DoShiftAP(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
-                      LT *Result) {
-  const unsigned Bits = LHS.bitWidth();
-  const APSInt &LHSAP = LHS.toAPSInt();
-  APSInt RHSAP = RHS.toAPSInt();
-
-  // OpenCL 6.3j: shift values are effectively % word size of LHS.
-  if (S.getLangOpts().OpenCL)
-    RHSAP &= APSInt(llvm::APInt(RHSAP.getBitWidth(),
-                                static_cast<uint64_t>(LHSAP.getBitWidth() - 1)),
-                    RHSAP.isUnsigned());
-
-  if (RHS.isNegative()) {
-    // During constant-folding, a negative shift is an opposite shift. Such a
-    // shift is not a constant expression.
-    const SourceInfo &Loc = S.Current->getSource(OpPC);
-    S.CCEDiag(Loc, diag::note_constexpr_negative_shift) << RHS.toAPSInt();
-    if (!S.noteUndefinedBehavior())
-      return false;
-    RHS = -RHS;
-    return DoShiftAP<LT, RT,
-                     Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
-        S, OpPC, LHS, RHS, Result);
-  }
-
-  if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
-    return false;
-
-  if constexpr (Dir == ShiftDir::Left) {
-    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
-    Result->copy(LHSAP << SA);
-  } else {
-    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
-    Result->copy(LHSAP >> SA);
-  }
-
-  S.Stk.push<LT>(*Result);
-  return true;
-}
-
 template <PrimType NameL, PrimType NameR>
 inline bool Shr(InterpState &S, CodePtr OpPC) {
   using LT = typename PrimConv<NameL>::T;
@@ -2837,13 +2659,7 @@ inline bool Shr(InterpState &S, CodePtr OpPC) {
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
 
-  if constexpr (needsAlloc<LT>()) {
-    LT Result = S.allocAP<LT>(LHS.bitWidth());
-    return DoShiftAP<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
-  } else {
-    LT Result;
-    return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
-  }
+  return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS);
 }
 
 template <PrimType NameL, PrimType NameR>
@@ -2852,13 +2668,8 @@ inline bool Shl(InterpState &S, CodePtr OpPC) {
   using RT = typename PrimConv<NameR>::T;
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
-  if constexpr (needsAlloc<LT>()) {
-    LT Result = S.allocAP<LT>(LHS.bitWidth());
-    return DoShiftAP<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
-  } else {
-    LT Result;
-    return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
-  }
+
+  return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS);
 }
 
 static inline bool ShiftFixedPoint(InterpState &S, CodePtr OpPC, bool Left) {
@@ -3441,15 +3252,7 @@ inline bool BitCastPrim(InterpState &S, CodePtr OpPC, bool TargetIsUCharOrByte,
 
     if constexpr (std::is_same_v<T, Floating>) {
       assert(Sem);
-      Floating Result = S.allocFloat(*Sem);
-      Floating::bitcastFromMemory(Buff.data(), *Sem, &Result);
-      S.Stk.push<Floating>(Result);
-
-      // S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
-    } else if constexpr (needsAlloc<T>()) {
-      T Result = S.allocAP<T>(ResultBitWidth);
-      T::bitcastFromMemory(Buff.data(), ResultBitWidth, &Result);
-      S.Stk.push<T>(Result);
+      S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
     } else {
       assert(!Sem);
       S.Stk.push<T>(T::bitcastFromMemory(Buff.data(), ResultBitWidth));
@@ -3507,11 +3310,7 @@ template <typename T> inline T ReadArg(InterpState &S, CodePtr &OpPC) {
 }
 
 template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
-  auto &Semantics =
-      llvm::APFloatBase::EnumToSemantics(Floating::deserializeSemantics(*OpPC));
-
-  auto F = S.allocFloat(Semantics);
-  Floating::deserialize(*OpPC, &F);
+  Floating F = Floating::deserialize(*OpPC);
   OpPC += align(F.bytesToSerialize());
   return F;
 }
@@ -3519,25 +3318,17 @@ template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
 template <>
 inline IntegralAP<false> ReadArg<IntegralAP<false>>(InterpState &S,
                                                     CodePtr &OpPC) {
-  uint32_t BitWidth = IntegralAP<false>::deserializeSize(*OpPC);
-  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
-  assert(Result.bitWidth() == BitWidth);
-
-  IntegralAP<false>::deserialize(*OpPC, &Result);
-  OpPC += align(Result.bytesToSerialize());
-  return Result;
+  IntegralAP<false> I = IntegralAP<false>::deserialize(*OpPC);
+  OpPC += align(I.bytesToSerialize());
+  return I;
 }
 
 template <>
 inline IntegralAP<true> ReadArg<IntegralAP<true>>(InterpState &S,
                                                   CodePtr &OpPC) {
-  uint32_t BitWidth = IntegralAP<true>::deserializeSize(*OpPC);
-  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
-  assert(Result.bitWidth() == BitWidth);
-
-  IntegralAP<true>::deserialize(*OpPC, &Result);
-  OpPC += align(Result.bytesToSerialize());
-  return Result;
+  IntegralAP<true> I = IntegralAP<true>::deserialize(*OpPC);
+  OpPC += align(I.bytesToSerialize());
+  return I;
 }
 
 template <>
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index 5304bd77f2c0..d01e3d042a8b 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -57,21 +57,6 @@ static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
   assert(T);
 
   unsigned BitWidth = S.getASTContext().getTypeSize(QT);
-
-  if (T == PT_IntAPS) {
-    auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
-    Result.copy(Val);
-    S.Stk.push<IntegralAP<true>>(Result);
-    return;
-  }
-
-  if (T == PT_IntAP) {
-    auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
-    Result.copy(Val);
-    S.Stk.push<IntegralAP<false>>(Result);
-    return;
-  }
-
   if (QT->isSignedIntegerOrEnumerationType()) {
     int64_t V = Val.getSExtValue();
     INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
@@ -342,13 +327,13 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  Floating Result = S.allocFloat(TargetSemantics);
+  Floating Result;
   if (S.getASTContext().getTargetInfo().isNan2008()) {
     if (Signaling)
-      Result.copy(
+      Result = Floating(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result.copy(
+      Result = Floating(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
   } else {
     // Prior to IEEE 754-2008, architectures were allowed to choose whether
@@ -357,10 +342,10 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
     // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
     // sNaN. This is now known as "legacy NaN" encoding.
     if (Signaling)
-      Result.copy(
+      Result = Floating(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result.copy(
+      Result = Floating(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
   }
 
@@ -375,9 +360,7 @@ static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  Floating Result = S.allocFloat(TargetSemantics);
-  Result.copy(APFloat::getInf(TargetSemantics));
-  S.Stk.push<Floating>(Result);
+  S.Stk.push<Floating>(Floating::getInf(TargetSemantics));
   return true;
 }
 
@@ -385,12 +368,10 @@ static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame) {
   const Floating &Arg2 = S.Stk.pop<Floating>();
   const Floating &Arg1 = S.Stk.pop<Floating>();
-  Floating Result = S.allocFloat(Arg1.getSemantics());
 
   APFloat Copy = Arg1.getAPFloat();
   Copy.copySign(Arg2.getAPFloat());
-  Result.copy(Copy);
-  S.Stk.push<Floating>(Result);
+  S.Stk.push<Floating>(Floating(Copy));
 
   return true;
 }
@@ -399,13 +380,11 @@ static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
-  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    S.Stk.push<Floating>(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
-  S.Stk.push<Floating>(Result);
+    S.Stk.push<Floating>(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
   return true;
 }
 
@@ -413,13 +392,11 @@ static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
-  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    S.Stk.push<Floating>(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
-  S.Stk.push<Floating>(Result);
+    S.Stk.push<Floating>(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
   return true;
 }
 
@@ -594,16 +571,8 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame) {
   const Floating &Val = S.Stk.pop<Floating>();
-  APFloat F = Val.getAPFloat();
-  if (!F.isNegative()) {
-    S.Stk.push<Floating>(Val);
-    return true;
-  }
 
-  Floating Result = S.allocFloat(Val.getSemantics());
-  F.changeSign();
-  Result.copy(F);
-  S.Stk.push<Floating>(Result);
+  S.Stk.push<Floating>(Floating::abs(Val));
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 2569cac018b3..239b3104e89f 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -402,9 +402,7 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
           if (llvm::sys::IsBigEndianHost)
             swapBytes(M.get(), NumBits.roundToBytes());
 
-          Floating R = S.allocFloat(Semantics);
-          Floating::bitcastFromMemory(M.get(), Semantics, &R);
-          P.deref<Floating>() = R;
+          P.deref<Floating>() = Floating::bitcastFromMemory(M.get(), Semantics);
           P.initialize();
           return true;
         }
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index 08765561985e..e8dc6f0483d6 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -15,7 +15,6 @@
 
 #include "Context.h"
 #include "DynamicAllocator.h"
-#include "Floating.h"
 #include "Function.h"
 #include "InterpFrame.h"
 #include "InterpStack.h"
@@ -127,33 +126,6 @@ public:
 
   StdAllocatorCaller getStdAllocatorCaller(StringRef Name) const;
 
-  void *allocate(size_t Size, unsigned Align = 8) const {
-    return Allocator.Allocate(Size, Align);
-  }
-  template <typename T> T *allocate(size_t Num = 1) const {
-    return static_cast<T *>(allocate(Num * sizeof(T), alignof(T)));
-  }
-
-  template <typename T> T allocAP(unsigned BitWidth) {
-    unsigned NumWords = APInt::getNumWords(BitWidth);
-    if (NumWords == 1)
-      return T(BitWidth);
-    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
-    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
-    return T(Mem, BitWidth);
-  }
-
-  Floating allocFloat(const llvm::fltSemantics &Sem) {
-    if (Floating::singleWord(Sem))
-      return Floating(llvm::APFloatBase::SemanticsToEnum(Sem));
-
-    unsigned NumWords =
-        APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem));
-    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
-    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
-    return Floating(Mem, llvm::APFloatBase::SemanticsToEnum(Sem));
-  }
-
 private:
   friend class EvaluationResult;
   friend class InterpStateCCOverride;
@@ -189,8 +161,6 @@ public:
   llvm::SmallVector<
       std::pair<const Expr *, const LifetimeExtendedTemporaryDecl *>>
       SeenGlobalTemporaries;
-
-  mutable llvm::BumpPtrAllocator Allocator;
 };
 
 class InterpStateCCOverride final {
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 57e01f7bd9da..c76ac5f8ae86 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -48,7 +48,6 @@ def ArgUint64 : ArgType { let Name = "uint64_t"; }
 def ArgIntAP : ArgType { let Name = "IntegralAP<false>"; let AsRef = true; }
 def ArgIntAPS : ArgType { let Name = "IntegralAP<true>"; let AsRef = true; }
 def ArgFloat : ArgType { let Name = "Floating"; let AsRef = true; }
-
 def ArgBool : ArgType { let Name = "bool"; }
 def ArgFixedPoint : ArgType { let Name = "FixedPoint"; let AsRef = true; }
 
@@ -89,9 +88,6 @@ def IntegerAndFixedTypeClass : TypeClass {
                Uint32, Sint64, Uint64, IntAP, IntAPS, FixedPoint];
 }
 
-def IntegralTypeClass : TypeClass {
-  let Types = !listconcat(IntegerTypeClass.Types, [Bool]);
-}
 def FixedSizeIntegralTypeClass : TypeClass {
   let Types = [Sint8, Uint8, Sint16, Uint16, Sint32,
                Uint32, Sint64, Uint64, Bool];
@@ -269,13 +265,12 @@ def ConstSint32 : ConstOpcode<Sint32, ArgSint32>;
 def ConstUint32 : ConstOpcode<Uint32, ArgUint32>;
 def ConstSint64 : ConstOpcode<Sint64, ArgSint64>;
 def ConstUint64 : ConstOpcode<Uint64, ArgUint64>;
-def ConstIntAP : ConstOpcode<IntAP, ArgIntAP>;
-def ConstIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
+def ConstFloat : ConstOpcode<Float, ArgFloat>;
+def constIntAP : ConstOpcode<IntAP, ArgIntAP>;
+def constIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
 def ConstBool : ConstOpcode<Bool, ArgBool>;
 def ConstFixedPoint : ConstOpcode<FixedPoint, ArgFixedPoint>;
 
-def ConstFloat : Opcode { let Args = [ArgFloat]; }
-
 // [] -> [Integer]
 def Zero : Opcode {
   let Types = [FixedSizeIntegralTypeClass];
@@ -333,7 +328,6 @@ def GetMemberPtrBasePop : Opcode {
 
 def FinishInitPop : Opcode;
 def FinishInit    : Opcode;
-def FinishInitGlobal : Opcode;
 
 def GetPtrDerivedPop : Opcode { let Args = [ArgUint32, ArgBool, ArgTypePtr]; }
 
@@ -395,7 +389,7 @@ class AccessOpcode : Opcode {
 }
 
 class BitFieldOpcode : Opcode {
-  let Types = [IntegralTypeClass];
+  let Types = [AluTypeClass];
   let Args = [ArgRecordField];
   let HasGroup = 1;
 }
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index a156cccbb3c1..6152fbfbe3a7 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -76,13 +76,6 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
 }
 
 constexpr bool isIntegralType(PrimType T) { return T <= PT_FixedPoint; }
-template <typename T> constexpr bool needsAlloc() {
-  return std::is_same_v<T, IntegralAP<false>> ||
-         std::is_same_v<T, IntegralAP<true>> || std::is_same_v<T, Floating>;
-}
-constexpr bool needsAlloc(PrimType T) {
-  return T == PT_IntAP || T == PT_IntAPS || T == PT_Float;
-}
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
@@ -216,16 +209,6 @@ static inline bool aligned(const void *P) {
     }                                                                          \
   } while (0)
 
-#define TYPE_SWITCH_ALLOC(Expr, B)                                             \
-  do {                                                                         \
-    switch (Expr) {                                                            \
-      TYPE_SWITCH_CASE(PT_Float, B)                                            \
-      TYPE_SWITCH_CASE(PT_IntAP, B)                                            \
-      TYPE_SWITCH_CASE(PT_IntAPS, B)                                           \
-    default:;                                                                  \
-    }                                                                          \
-  } while (0)
-
 #define COMPOSITE_TYPE_SWITCH(Expr, B, D)                                      \
   do {                                                                         \
     switch (Expr) {                                                            \
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index 5d9c42244749..23ba1bbd193b 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -132,14 +132,6 @@ public:
                                bool IsMutable = false, bool IsVolatile = false,
                                const Expr *Init = nullptr);
 
-  void *Allocate(size_t Size, unsigned Align = 8) const {
-    return Allocator.Allocate(Size, Align);
-  }
-  template <typename T> T *Allocate(size_t Num = 1) const {
-    return static_cast<T *>(Allocate(Num * sizeof(T), alignof(T)));
-  }
-  void Deallocate(void *Ptr) const {}
-
   /// Context to manage declaration lifetimes.
   class DeclScope {
   public:
@@ -212,7 +204,7 @@ private:
   };
 
   /// Allocator for globals.
-  mutable PoolAllocTy Allocator;
+  PoolAllocTy Allocator;
 
   /// Global objects.
   std::vector<Global *> Globals;
@@ -246,18 +238,4 @@ public:
 } // namespace interp
 } // namespace clang
 
-inline void *operator new(size_t Bytes, const clang::interp::Program &C,
-                          size_t Alignment = 8) {
-  return C.Allocate(Bytes, Alignment);
-}
-
-inline void operator delete(void *Ptr, const clang::interp::Program &C,
-                            size_t) {
-  C.Deallocate(Ptr);
-}
-inline void *operator new[](size_t Bytes, const clang::interp::Program &C,
-                            size_t Alignment = 8) {
-  return C.Allocate(Bytes, Alignment);
-}
-
 #endif
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
index 1013a771d13b..710612bef8fd 100644
--- a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
+++ b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
@@ -21,9 +21,6 @@ template <class To, class From>
 constexpr To bit_cast(const From &from) {
   static_assert(sizeof(To) == sizeof(From));
   return __builtin_bit_cast(To, from);
-#if __x86_64
-  // both-note@-2 {{indeterminate value can only initialize an object of type}}
-#endif
 }
 
 template <class Intermediate, class Init>
@@ -41,8 +38,11 @@ constexpr Init round_trip(const Init &init) {
 
 namespace test_long_double {
 #if __x86_64
-constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // both-error{{must be initialized by a constant expression}}\
-                                                                                 // both-note{{in call}}
+/// FIXME: We could enable this, but since it aborts, it causes the usual mempory leak.
+#if 0
+constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // expected-error{{must be initialized by a constant expression}}\
+                                                                                 // expected-note{{in call}}
+#endif
 constexpr long double ld = 3.1425926539;
 
 struct bytes {
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 174c1ffa79a4..21dca15a4577 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -208,7 +208,7 @@ namespace nan {
 
   constexpr double NaN3 = __builtin_nan("foo"); // both-error {{must be initialized by a constant expression}}
   constexpr float NaN4 = __builtin_nanf("");
-  constexpr long double NaN5 = __builtin_nanf128("");
+  //constexpr long double NaN5 = __builtin_nanf128("");
 
   /// FIXME: This should be accepted by the current interpreter as well.
   constexpr char f[] = {'0', 'x', 'A', 'E', '\0'};
@@ -655,6 +655,8 @@ void test_noexcept(int *i) {
 } // end namespace test_launder
 
 
+/// FIXME: The commented out tests here use a IntAP value and fail.
+/// This currently means we will leak the IntAP value since nothing cleans it up.
 namespace clz {
   char clz1[__builtin_clz(1) == BITSIZE(int) - 1 ? 1 : -1];
   char clz2[__builtin_clz(7) == BITSIZE(int) - 3 ? 1 : -1];
@@ -707,7 +709,7 @@ namespace clz {
   char clz48[__builtin_clzg(1ULL << (BITSIZE(long long) - 1)) == 0 ? 1 : -1];
   char clz49[__builtin_clzg(1ULL << (BITSIZE(long long) - 1), 42) == 0 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  int clz50 = __builtin_clzg((unsigned __int128)0);
+  // int clz50 = __builtin_clzg((unsigned __int128)0);
   char clz51[__builtin_clzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char clz52[__builtin_clzg((unsigned __int128)0x1) == BITSIZE(__int128) - 1 ? 1 : -1];
   char clz53[__builtin_clzg((unsigned __int128)0x1, 42) == BITSIZE(__int128) - 1 ? 1 : -1];
@@ -715,7 +717,7 @@ namespace clz {
   char clz55[__builtin_clzg((unsigned __int128)0xf, 42) == BITSIZE(__int128) - 4 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
+  // int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
   char clz59[__builtin_clzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char clz60[__builtin_clzg((unsigned _BitInt(128))0x1) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
   char clz61[__builtin_clzg((unsigned _BitInt(128))0x1, 42) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
@@ -773,7 +775,7 @@ namespace ctz {
   char ctz46[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1)) == BITSIZE(long long) - 1 ? 1 : -1];
   char ctz47[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1), 42) == BITSIZE(long long) - 1 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  int ctz48 = __builtin_ctzg((unsigned __int128)0);
+  // int ctz48 = __builtin_ctzg((unsigned __int128)0);
   char ctz49[__builtin_ctzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char ctz50[__builtin_ctzg((unsigned __int128)0x1) == 0 ? 1 : -1];
   char ctz51[__builtin_ctzg((unsigned __int128)0x1, 42) == 0 ? 1 : -1];
@@ -783,7 +785,7 @@ namespace ctz {
   char ctz55[__builtin_ctzg((unsigned __int128)1 << (BITSIZE(__int128) - 1), 42) == BITSIZE(__int128) - 1 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
+  // int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
   char ctz57[__builtin_ctzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char ctz58[__builtin_ctzg((unsigned _BitInt(128))0x1) == 0 ? 1 : -1];
   char ctz59[__builtin_ctzg((unsigned _BitInt(128))0x1, 42) == 0 ? 1 : -1];
-- 
cgit v1.2.3


From 6729da647afa2b0ee040ccd4f06153e45d6ca738 Mon Sep 17 00:00:00 2001
From: Kunwar Grover <groverkss@gmail.com>
Date: Wed, 18 Jun 2025 14:19:17 +0100
Subject: [mlir][amdgpu][nfc] Add PatternBenefit to populate methods (#144663)

---
 mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h          | 10 +++++++---
 mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp         |  5 +++--
 mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp |  4 ++--
 mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp     |  4 ++--
 4 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
index 94dd9e3a2933..a52ee2ee89ca 100644
--- a/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/AMDGPU/Transforms/Passes.h
@@ -14,6 +14,7 @@
 #define MLIR_DIALECT_AMDGPU_TRANSFORMS_PASSES_H_
 
 #include "mlir/Dialect/AMDGPU/Utils/Chipset.h"
+#include "mlir/IR/PatternMatch.h"
 #include "mlir/Pass/Pass.h"
 
 namespace mlir {
@@ -28,11 +29,14 @@ namespace amdgpu {
 
 void populateAmdgpuEmulateAtomicsPatterns(ConversionTarget &target,
                                           RewritePatternSet &patterns,
-                                          Chipset chipset);
+                                          Chipset chipset,
+                                          PatternBenefit benefit = 1);
 
-void populateAmdgpuResolveStridedMetadataPatterns(RewritePatternSet &patterns);
+void populateAmdgpuResolveStridedMetadataPatterns(RewritePatternSet &patterns,
+                                                  PatternBenefit benefit = 1);
 
-void populateAmdgpuTransferReadToLoadPatterns(RewritePatternSet &patterns);
+void populateAmdgpuTransferReadToLoadPatterns(RewritePatternSet &patterns,
+                                              PatternBenefit benefit = 1);
 
 } // namespace amdgpu
 } // namespace mlir
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
index 7dd4be66d2bd..fd2ba0683786 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/EmulateAtomics.cpp
@@ -164,7 +164,8 @@ LogicalResult RawBufferAtomicByCasPattern<AtomicOp, ArithOp>::matchAndRewrite(
 }
 
 void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
-    ConversionTarget &target, RewritePatternSet &patterns, Chipset chipset) {
+    ConversionTarget &target, RewritePatternSet &patterns, Chipset chipset,
+    PatternBenefit benefit) {
   // gfx10 has no atomic adds.
   if (chipset.majorVersion == 10 || chipset < Chipset(9, 0, 8)) {
     target.addIllegalOp<RawBufferAtomicFaddOp>();
@@ -204,7 +205,7 @@ void mlir::amdgpu::populateAmdgpuEmulateAtomicsPatterns(
       RawBufferAtomicByCasPattern<RawBufferAtomicFmaxOp, arith::MaximumFOp>,
       RawBufferAtomicByCasPattern<RawBufferAtomicSmaxOp, arith::MaxSIOp>,
       RawBufferAtomicByCasPattern<RawBufferAtomicUminOp, arith::MinUIOp>>(
-      patterns.getContext());
+      patterns.getContext(), benefit);
 }
 
 void AmdgpuEmulateAtomicsPass::runOnOperation() {
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
index 4b3d94b4ce2a..195f59d62555 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/ResolveStridedMetadata.cpp
@@ -66,9 +66,9 @@ struct ExtractStridedMetadataOnFatRawBufferCastFolder final
 } // namespace
 
 void mlir::amdgpu::populateAmdgpuResolveStridedMetadataPatterns(
-    RewritePatternSet &patterns) {
+    RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<ExtractStridedMetadataOnFatRawBufferCastFolder>(
-      patterns.getContext());
+      patterns.getContext(), benefit);
 }
 
 void AmdgpuResolveStridedMetadataPass::runOnOperation() {
diff --git a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
index 96925dbf9f28..f5b12a9524cc 100644
--- a/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Transforms/TransferReadToLoad.cpp
@@ -222,8 +222,8 @@ struct TransferReadLowering final : OpRewritePattern<vector::TransferReadOp> {
 } // namespace
 
 void mlir::amdgpu::populateAmdgpuTransferReadToLoadPatterns(
-    RewritePatternSet &patterns) {
-  patterns.add<TransferReadLowering>(patterns.getContext());
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<TransferReadLowering>(patterns.getContext(), benefit);
 }
 
 struct AmdgpuTransferReadToLoadPass final
-- 
cgit v1.2.3


From c4d99704e22097703c57ee67baea96fdabfd68ab Mon Sep 17 00:00:00 2001
From: Garvit Gupta <quic_garvgupt@quicinc.com>
Date: Wed, 18 Jun 2025 18:53:45 +0530
Subject: =?UTF-8?q?Revert=20"Reland=20[Driver]=20Add=20support=20for=20GCC?=
 =?UTF-8?q?=20installation=20detection=20in=E2=80=A6=20(#144684)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… Baremetal toolchain (#144640)"

This reverts commit 45ea46c44636094e9fcdbbeabfd11f9d0fad5e38.
---
 clang/docs/Toolchain.rst                           |   5 -
 clang/include/clang/Basic/DiagnosticDriverKinds.td |   3 -
 clang/lib/Driver/ToolChains/BareMetal.cpp          | 235 ++++++---------------
 clang/lib/Driver/ToolChains/BareMetal.h            |  19 +-
 .../aarch64-none-elf/include/c++/8.2.1/.keep       |   0
 .../aarch64-none-elf/lib/.keep                     |   0
 .../aarch64-none-elf/lib/crt0.o                    |   0
 .../basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld |   1 -
 .../lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o      |   0
 .../lib/gcc/aarch64-none-elf/8.2.1/crtend.o        |   0
 .../aarch64-none-elf/lib/crt0.o                    |   0
 .../aarch64-none-elf/lib/crtbegin.o                |   0
 .../aarch64-none-elf/lib/crtend.o                  |   0
 .../bin/aarch64-none-elf-ld                        |   1 -
 .../armv6m-none-eabi/include/c++/8.2.1/.keep       |   0
 .../basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep  |   0
 .../basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o |   0
 .../basic_arm_gcc_tree/bin/armv6m-none-eabi-ld     |   1 -
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o      |   0
 .../lib/gcc/armv6m-none-eabi/8.2.1/crtend.o        |   0
 .../armv6m-none-eabi/lib/crt0.o                    |   0
 .../armv6m-none-eabi/lib/crtbegin.o                |   0
 .../armv6m-none-eabi/lib/crtend.o                  |   0
 .../basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld   |   1 -
 clang/test/Driver/aarch64-gnutools.c               |   4 -
 clang/test/Driver/aarch64-toolchain-extra.c        |  29 ---
 clang/test/Driver/aarch64-toolchain.c              |  62 ------
 clang/test/Driver/arm-gnutools.c                   |   6 -
 clang/test/Driver/arm-toolchain-extra.c            |  30 ---
 clang/test/Driver/arm-toolchain.c                  |  63 ------
 clang/test/Driver/baremetal.cpp                    |  16 --
 clang/test/Driver/check-no-multlib-warning.c       |  10 -
 32 files changed, 64 insertions(+), 422 deletions(-)
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
 delete mode 100755 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
 delete mode 100644 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
 delete mode 100755 clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
 delete mode 100755 clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
 delete mode 100644 clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
 delete mode 100755 clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
 delete mode 100644 clang/test/Driver/aarch64-gnutools.c
 delete mode 100644 clang/test/Driver/aarch64-toolchain-extra.c
 delete mode 100644 clang/test/Driver/aarch64-toolchain.c
 delete mode 100644 clang/test/Driver/arm-gnutools.c
 delete mode 100644 clang/test/Driver/arm-toolchain-extra.c
 delete mode 100644 clang/test/Driver/arm-toolchain.c
 delete mode 100644 clang/test/Driver/check-no-multlib-warning.c

diff --git a/clang/docs/Toolchain.rst b/clang/docs/Toolchain.rst
index d56b21d74c7e..958199eb7a2e 100644
--- a/clang/docs/Toolchain.rst
+++ b/clang/docs/Toolchain.rst
@@ -347,8 +347,3 @@ workarounds for issues discovered in libstdc++, and these are removed
 as fixed libstdc++ becomes sufficiently old.
 
 You can instruct Clang to use libstdc++ with the ``-stdlib=libstdc++`` flag.
-
-GCC Installation
-=================
-Users can point to their GCC installation by using the ``-gcc-toolchain`` or by
-using ``-gcc-install-dir`` flag.
diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td
index 94224e103875..29f6480ba935 100644
--- a/clang/include/clang/Basic/DiagnosticDriverKinds.td
+++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td
@@ -847,9 +847,6 @@ def note_drv_available_multilibs : Note<
   "available multilibs are:%0">;
 def err_drv_multilib_custom_error : Error<
   "multilib configuration error: %0">;
-def warn_drv_multilib_not_available_for_target: Warning<
-  "no multilib structure encoded for Arm, Aarch64 and PPC targets">,
-  InGroup<DiagGroup<"multilib-not-found">>;
 
 def err_drv_experimental_crel : Error<
   "-Wa,--allow-experimental-crel must be specified to use -Wa,--crel. "
diff --git a/clang/lib/Driver/ToolChains/BareMetal.cpp b/clang/lib/Driver/ToolChains/BareMetal.cpp
index 0fbfe6c77f34..d8168ed15feb 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.cpp
+++ b/clang/lib/Driver/ToolChains/BareMetal.cpp
@@ -31,40 +31,6 @@ using namespace clang::driver;
 using namespace clang::driver::tools;
 using namespace clang::driver::toolchains;
 
-/// Is the triple {aarch64.aarch64_be}-none-elf?
-static bool isAArch64BareMetal(const llvm::Triple &Triple) {
-  if (Triple.getArch() != llvm::Triple::aarch64 &&
-      Triple.getArch() != llvm::Triple::aarch64_be)
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
-
-  return Triple.getEnvironmentName() == "elf";
-}
-
-static bool isRISCVBareMetal(const llvm::Triple &Triple) {
-  if (!Triple.isRISCV())
-    return false;
-
-  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
-    return false;
-
-  if (Triple.getOS() != llvm::Triple::UnknownOS)
-    return false;
-
-  return Triple.getEnvironmentName() == "elf";
-}
-
-/// Is the triple powerpc[64][le]-*-none-eabi?
-static bool isPPCBareMetal(const llvm::Triple &Triple) {
-  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
-         Triple.getEnvironment() == llvm::Triple::EABI;
-}
-
 static bool findRISCVMultilibs(const Driver &D,
                                const llvm::Triple &TargetTriple,
                                const ArgList &Args, DetectedMultilibs &Result) {
@@ -129,8 +95,7 @@ static bool findRISCVMultilibs(const Driver &D,
   return false;
 }
 
-static std::string computeClangRuntimesSysRoot(const Driver &D,
-                                               bool IncludeTriple) {
+static std::string computeBaseSysRoot(const Driver &D, bool IncludeTriple) {
   if (!D.SysRoot.empty())
     return D.SysRoot;
 
@@ -143,123 +108,56 @@ static std::string computeClangRuntimesSysRoot(const Driver &D,
   return std::string(SysRootDir);
 }
 
-// Only consider the GCC toolchain based on the values provided through the
-// `--gcc-toolchain` and `--gcc-install-dir` flags. The function below returns
-// whether the GCC toolchain was initialized successfully.
-bool BareMetal::initGCCInstallation(const llvm::Triple &Triple,
-                                    const llvm::opt::ArgList &Args) {
-  if (Args.getLastArg(options::OPT_gcc_toolchain) ||
-      Args.getLastArg(clang::driver::options::OPT_gcc_install_dir_EQ)) {
-    GCCInstallation.init(Triple, Args);
-    return GCCInstallation.isValid();
+BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
+                     const ArgList &Args)
+    : ToolChain(D, Triple, Args),
+      SysRoot(computeBaseSysRoot(D, /*IncludeTriple=*/true)) {
+  getProgramPaths().push_back(getDriver().Dir);
+
+  findMultilibs(D, Triple, Args);
+  SmallString<128> SysRoot(computeSysRoot());
+  if (!SysRoot.empty()) {
+    for (const Multilib &M : getOrderedMultilibs()) {
+      SmallString<128> Dir(SysRoot);
+      llvm::sys::path::append(Dir, M.osSuffix(), "lib");
+      getFilePaths().push_back(std::string(Dir));
+      getLibraryPaths().push_back(std::string(Dir));
+    }
   }
-  return false;
 }
 
-// This logic is adapted from RISCVToolChain.cpp as part of the ongoing effort
-// to merge RISCVToolChain into the Baremetal toolchain. It infers the presence
-// of a valid GCC toolchain by checking whether the `crt0.o` file exists in the
-// `bin/../<target-triple>/lib` directory.
-static bool detectGCCToolchainAdjacent(const Driver &D) {
-  SmallString<128> GCCDir;
-  llvm::sys::path::append(GCCDir, D.Dir, "..", D.getTargetTriple(),
-                          "lib/crt0.o");
-  return llvm::sys::fs::exists(GCCDir);
-}
+/// Is the triple {aarch64.aarch64_be}-none-elf?
+static bool isAArch64BareMetal(const llvm::Triple &Triple) {
+  if (Triple.getArch() != llvm::Triple::aarch64 &&
+      Triple.getArch() != llvm::Triple::aarch64_be)
+    return false;
 
-// If no sysroot is provided the driver will first attempt to infer it from the
-// values of `--gcc-install-dir` or `--gcc-toolchain`, which specify the
-// location of a GCC toolchain.
-// If neither flag is used, the sysroot defaults to either:
-//    - `bin/../<target-triple>`
-//    - `bin/../lib/clang-runtimes/<target-triple>`
-//
-// To use the `clang-runtimes` path, ensure that `../<target-triple>/lib/crt0.o`
-// does not exist relative to the driver.
-std::string BareMetal::computeSysRoot() const {
-  // Use Baremetal::sysroot if it has already been set.
-  if (!SysRoot.empty())
-    return SysRoot;
-
-  // Use the sysroot specified via the `--sysroot` command-line flag, if
-  // provided.
-  const Driver &D = getDriver();
-  if (!D.SysRoot.empty())
-    return D.SysRoot;
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
 
-  // Attempt to infer sysroot from a valid GCC installation.
-  // If no valid GCC installation, check for a GCC toolchain alongside Clang.
-  SmallString<128> inferredSysRoot;
-  if (IsGCCInstallationValid) {
-    llvm::sys::path::append(inferredSysRoot, GCCInstallation.getParentLibPath(),
-                            "..", GCCInstallation.getTriple().str());
-  } else if (detectGCCToolchainAdjacent(D)) {
-    // Use the triple as provided to the driver. Unlike the parsed triple
-    // this has not been normalized to always contain every field.
-    llvm::sys::path::append(inferredSysRoot, D.Dir, "..", D.getTargetTriple());
-  }
-  // If a valid sysroot was inferred and exists, use it
-  if (!inferredSysRoot.empty() && llvm::sys::fs::exists(inferredSysRoot))
-    return std::string(inferredSysRoot);
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
 
-  // Use the clang-runtimes path.
-  return computeClangRuntimesSysRoot(D, /*IncludeTriple*/ true);
+  return Triple.getEnvironmentName() == "elf";
 }
 
-static void addMultilibsFilePaths(const Driver &D, const MultilibSet &Multilibs,
-                                  const Multilib &Multilib,
-                                  StringRef InstallPath,
-                                  ToolChain::path_list &Paths) {
-  if (const auto &PathsCallback = Multilibs.filePathsCallback())
-    for (const auto &Path : PathsCallback(Multilib))
-      addPathIfExists(D, InstallPath + Path, Paths);
+static bool isRISCVBareMetal(const llvm::Triple &Triple) {
+  if (!Triple.isRISCV())
+    return false;
+
+  if (Triple.getVendor() != llvm::Triple::UnknownVendor)
+    return false;
+
+  if (Triple.getOS() != llvm::Triple::UnknownOS)
+    return false;
+
+  return Triple.getEnvironmentName() == "elf";
 }
 
-// GCC mutltilibs will only work for those targets that have their multlib
-// structure encoded into GCCInstallation. Baremetal toolchain supports ARM,
-// AArch64, RISCV and PPC and of these only RISCV have GCC multilibs hardcoded
-// in GCCInstallation.
-BareMetal::BareMetal(const Driver &D, const llvm::Triple &Triple,
-                     const ArgList &Args)
-    : Generic_ELF(D, Triple, Args) {
-  IsGCCInstallationValid = initGCCInstallation(Triple, Args);
-  std::string ComputedSysRoot = computeSysRoot();
-  if (IsGCCInstallationValid) {
-    if (!isRISCVBareMetal(Triple))
-      D.Diag(clang::diag::warn_drv_multilib_not_available_for_target);
-
-    Multilibs = GCCInstallation.getMultilibs();
-    SelectedMultilibs.assign({GCCInstallation.getMultilib()});
-
-    path_list &Paths = getFilePaths();
-    // Add toolchain/multilib specific file paths.
-    addMultilibsFilePaths(D, Multilibs, SelectedMultilibs.back(),
-                          GCCInstallation.getInstallPath(), Paths);
-    // Adding filepath for locating crt{begin,end}.o files.
-    Paths.push_back(GCCInstallation.getInstallPath().str());
-    // Adding filepath for locating crt0.o file.
-    Paths.push_back(ComputedSysRoot + "/lib");
-
-    ToolChain::path_list &PPaths = getProgramPaths();
-    // Multilib cross-compiler GCC installations put ld in a triple-prefixed
-    // directory off of the parent of the GCC installation.
-    PPaths.push_back(Twine(GCCInstallation.getParentLibPath() + "/../" +
-                           GCCInstallation.getTriple().str() + "/bin")
-                         .str());
-    PPaths.push_back((GCCInstallation.getParentLibPath() + "/../bin").str());
-  } else {
-    getProgramPaths().push_back(getDriver().Dir);
-    findMultilibs(D, Triple, Args);
-    const SmallString<128> SysRootDir(computeSysRoot());
-    if (!SysRootDir.empty()) {
-      for (const Multilib &M : getOrderedMultilibs()) {
-        SmallString<128> Dir(SysRootDir);
-        llvm::sys::path::append(Dir, M.osSuffix(), "lib");
-        getFilePaths().push_back(std::string(Dir));
-        getLibraryPaths().push_back(std::string(Dir));
-      }
-    }
-  }
+/// Is the triple powerpc[64][le]-*-none-eabi?
+static bool isPPCBareMetal(const llvm::Triple &Triple) {
+  return Triple.isPPC() && Triple.getOS() == llvm::Triple::UnknownOS &&
+         Triple.getEnvironment() == llvm::Triple::EABI;
 }
 
 static void
@@ -318,7 +216,7 @@ getMultilibConfigPath(const Driver &D, const llvm::Triple &Triple,
       return {};
     }
   } else {
-    MultilibPath = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
+    MultilibPath = computeBaseSysRoot(D, /*IncludeTriple=*/false);
     llvm::sys::path::append(MultilibPath, MultilibFilename);
   }
   return MultilibPath;
@@ -336,7 +234,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
   if (D.getVFS().exists(*MultilibPath)) {
     // If multilib.yaml is found, update sysroot so it doesn't use a target
     // specific suffix
-    SysRoot = computeClangRuntimesSysRoot(D, /*IncludeTriple=*/false);
+    SysRoot = computeBaseSysRoot(D, /*IncludeTriple=*/false);
     SmallVector<StringRef> CustomFlagMacroDefines;
     findMultilibsFromYAML(*this, D, *MultilibPath, Args, Result,
                           CustomFlagMacroDefines);
@@ -344,7 +242,7 @@ void BareMetal::findMultilibs(const Driver &D, const llvm::Triple &Triple,
     Multilibs = Result.Multilibs;
     MultilibMacroDefines.append(CustomFlagMacroDefines.begin(),
                                 CustomFlagMacroDefines.end());
-  } else if (isRISCVBareMetal(Triple) && !detectGCCToolchainAdjacent(D)) {
+  } else if (isRISCVBareMetal(Triple)) {
     if (findRISCVMultilibs(D, Triple, Args, Result)) {
       SelectedMultilibs = Result.SelectedMultilibs;
       Multilibs = Result.Multilibs;
@@ -365,6 +263,8 @@ Tool *BareMetal::buildStaticLibTool() const {
   return new tools::baremetal::StaticLibTool(*this);
 }
 
+std::string BareMetal::computeSysRoot() const { return SysRoot; }
+
 BareMetal::OrderedMultilibs BareMetal::getOrderedMultilibs() const {
   // Get multilibs in reverse order because they're ordered most-specific last.
   if (!SelectedMultilibs.empty())
@@ -392,10 +292,10 @@ void BareMetal::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
   if (std::optional<std::string> Path = getStdlibIncludePath())
     addSystemInclude(DriverArgs, CC1Args, *Path);
 
-  const SmallString<128> SysRootDir(computeSysRoot());
-  if (!SysRootDir.empty()) {
+  const SmallString<128> SysRoot(computeSysRoot());
+  if (!SysRoot.empty()) {
     for (const Multilib &M : getOrderedMultilibs()) {
-      SmallString<128> Dir(SysRootDir);
+      SmallString<128> Dir(SysRoot);
       llvm::sys::path::append(Dir, M.includeSuffix());
       llvm::sys::path::append(Dir, "include");
       addSystemInclude(DriverArgs, CC1Args, Dir.str());
@@ -409,19 +309,6 @@ void BareMetal::addClangTargetOptions(const ArgList &DriverArgs,
   CC1Args.push_back("-nostdsysteminc");
 }
 
-void BareMetal::addLibStdCxxIncludePaths(
-    const llvm::opt::ArgList &DriverArgs,
-    llvm::opt::ArgStringList &CC1Args) const {
-  if (!IsGCCInstallationValid)
-    return;
-  const GCCVersion &Version = GCCInstallation.getVersion();
-  StringRef TripleStr = GCCInstallation.getTriple().str();
-  const Multilib &Multilib = GCCInstallation.getMultilib();
-  addLibStdCXXIncludePaths(computeSysRoot() + "/include/c++/" + Version.Text,
-                           TripleStr, Multilib.includeSuffix(), DriverArgs,
-                           CC1Args);
-}
-
 void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
                                              ArgStringList &CC1Args) const {
   if (DriverArgs.hasArg(options::OPT_nostdinc, options::OPT_nostdlibinc,
@@ -452,23 +339,23 @@ void BareMetal::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   };
 
   switch (GetCXXStdlibType(DriverArgs)) {
-  case ToolChain::CST_Libcxx: {
-    SmallString<128> P(D.Dir);
-    llvm::sys::path::append(P, "..", "include");
-    AddCXXIncludePath(P);
-    break;
-  }
-  case ToolChain::CST_Libstdcxx:
-    addLibStdCxxIncludePaths(DriverArgs, CC1Args);
-    break;
+    case ToolChain::CST_Libcxx: {
+      SmallString<128> P(D.Dir);
+      llvm::sys::path::append(P, "..", "include");
+      AddCXXIncludePath(P);
+      break;
+    }
+    case ToolChain::CST_Libstdcxx:
+      // We only support libc++ toolchain installation.
+      break;
   }
 
-  std::string SysRootDir(computeSysRoot());
-  if (SysRootDir.empty())
+  std::string SysRoot(computeSysRoot());
+  if (SysRoot.empty())
     return;
 
   for (const Multilib &M : getOrderedMultilibs()) {
-    SmallString<128> Dir(SysRootDir);
+    SmallString<128> Dir(SysRoot);
     llvm::sys::path::append(Dir, M.gccSuffix());
     switch (GetCXXStdlibType(DriverArgs)) {
     case ToolChain::CST_Libcxx: {
diff --git a/clang/lib/Driver/ToolChains/BareMetal.h b/clang/lib/Driver/ToolChains/BareMetal.h
index 930f8584e643..f6295bda0a6a 100644
--- a/clang/lib/Driver/ToolChains/BareMetal.h
+++ b/clang/lib/Driver/ToolChains/BareMetal.h
@@ -9,7 +9,6 @@
 #ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 #define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_BAREMETAL_H
 
-#include "ToolChains/Gnu.h"
 #include "clang/Driver/Tool.h"
 #include "clang/Driver/ToolChain.h"
 
@@ -20,7 +19,7 @@ namespace driver {
 
 namespace toolchains {
 
-class LLVM_LIBRARY_VISIBILITY BareMetal : public Generic_ELF {
+class LLVM_LIBRARY_VISIBILITY BareMetal : public ToolChain {
 public:
   BareMetal(const Driver &D, const llvm::Triple &Triple,
             const llvm::opt::ArgList &Args);
@@ -36,8 +35,7 @@ protected:
   Tool *buildStaticLibTool() const override;
 
 public:
-  bool initGCCInstallation(const llvm::Triple &Triple,
-                           const llvm::opt::ArgList &Args);
+  bool useIntegratedAs() const override { return true; }
   bool isBareMetal() const override { return true; }
   bool isCrossCompiling() const override { return true; }
   bool HasNativeLLVMSupport() const override { return true; }
@@ -50,15 +48,9 @@ public:
 
   StringRef getOSLibName() const override { return "baremetal"; }
 
-  UnwindTableLevel
-  getDefaultUnwindTableLevel(const llvm::opt::ArgList &Args) const override {
-    return UnwindTableLevel::None;
-  }
-
   RuntimeLibType GetDefaultRuntimeLibType() const override {
     return ToolChain::RLT_CompilerRT;
   }
-
   CXXStdlibType GetDefaultCXXStdlibType() const override {
     return ToolChain::CST_Libcxx;
   }
@@ -75,9 +67,6 @@ public:
   void AddClangCXXStdlibIncludeArgs(
       const llvm::opt::ArgList &DriverArgs,
       llvm::opt::ArgStringList &CC1Args) const override;
-  void
-  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
-                           llvm::opt::ArgStringList &CC1Args) const override;
   std::string computeSysRoot() const override;
   SanitizerMask getSupportedSanitizers() const override;
 
@@ -91,8 +80,6 @@ private:
 
   std::string SysRoot;
 
-  bool IsGCCInstallationValid;
-
   SmallVector<std::string> MultilibMacroDefines;
 };
 
@@ -117,7 +104,7 @@ public:
 
 class LLVM_LIBRARY_VISIBILITY Linker final : public Tool {
 public:
-  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "linker", TC) {}
+  Linker(const ToolChain &TC) : Tool("baremetal::Linker", "ld.lld", TC) {}
   bool isLinkJob() const override { return true; }
   bool hasIntegratedCPP() const override { return false; }
   void ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/.keep
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/.keep
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/lib/crt0.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
deleted file mode 100755
index b23e55619b2f..000000000000
--- a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/bin/aarch64-none-elf-ld
+++ /dev/null
@@ -1 +0,0 @@
-#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtbegin.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/crtend.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crt0.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtbegin.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf/lib/crtend.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld b/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
deleted file mode 100755
index b23e55619b2f..000000000000
--- a/clang/test/Driver/Inputs/basic_aarch64_nogcc_tree/bin/aarch64-none-elf-ld
+++ /dev/null
@@ -1 +0,0 @@
-#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/.keep
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/.keep
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/lib/crt0.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
deleted file mode 100755
index b23e55619b2f..000000000000
--- a/clang/test/Driver/Inputs/basic_arm_gcc_tree/bin/armv6m-none-eabi-ld
+++ /dev/null
@@ -1 +0,0 @@
-#!/bin/true
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtbegin.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o b/clang/test/Driver/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/crtend.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crt0.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtbegin.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi/lib/crtend.o
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld b/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
deleted file mode 100755
index b23e55619b2f..000000000000
--- a/clang/test/Driver/Inputs/basic_arm_nogcc_tree/bin/armv6m-none-eabi-ld
+++ /dev/null
@@ -1 +0,0 @@
-#!/bin/true
diff --git a/clang/test/Driver/aarch64-gnutools.c b/clang/test/Driver/aarch64-gnutools.c
deleted file mode 100644
index 0214639ed380..000000000000
--- a/clang/test/Driver/aarch64-gnutools.c
+++ /dev/null
@@ -1,4 +0,0 @@
-// RUN: %clang --target=aarch64-none-elf  --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -fno-integrated-as %s -### -c \
-// RUN: 2>&1 | FileCheck %s
-
-// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/aarch64-toolchain-extra.c b/clang/test/Driver/aarch64-toolchain-extra.c
deleted file mode 100644
index eb8c741ae1ad..000000000000
--- a/clang/test/Driver/aarch64-toolchain-extra.c
+++ /dev/null
@@ -1,29 +0,0 @@
-// A basic clang -cc1 command-line, and simple environment check.
-
-// The tests here are similar to those in aarch64-toolchain.c, however
-// these tests need to create symlinks to test directory trees in order to
-// set up the environment and therefore shell support is required.
-// XFAIL: target={{.*}}-fuchsia{{.*}}
-// REQUIRES: shell
-// UNSUPPORTED: system-windows
-
-// If there is no GCC install detected then the driver searches for executables
-// and runtime starting from the directory tree above the driver itself.
-// The test below checks that the driver correctly finds the linker and
-// runtime if and only if they exist.
-//
-// RUN: rm -rf %t
-// RUN: mkdir -p %t/aarch64-nogcc/bin
-// RUN: ln -s %clang %t/aarch64-nogcc/bin/clang
-// RUN: ln -s %S/Inputs/basic_aarch64_nogcc_tree/aarch64-none-elf %t/aarch64-nogcc/aarch64-none-elf
-// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    --gcc-toolchain=%t/aarch64-nogcc/invalid \
-// RUN:    --target=aarch64-none-elf --rtlib=libgcc -fuse-ld=ld 2>&1 \
-// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
-
-// RUN: %t/aarch64-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    --sysroot=%t/aarch64-nogcc/bin/../aarch64-none-elf \
-// RUN:    --target=aarch64-none-elf --rtlib=libgcc -fuse-ld=ld 2>&1 \
-// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
-
-// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/aarch64-nogcc/bin/../aarch64-none-elf/include"
diff --git a/clang/test/Driver/aarch64-toolchain.c b/clang/test/Driver/aarch64-toolchain.c
deleted file mode 100644
index 74841eec598b..000000000000
--- a/clang/test/Driver/aarch64-toolchain.c
+++ /dev/null
@@ -1,62 +0,0 @@
-// XFAIL: target={{.*}}-fuchsia{{.*}}
-// UNSUPPORTED: system-windows
-
-// RUN: %clang -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
-// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL %s
-
-// C-AARCH64-BAREMETAL: "-cc1" "-triple" "aarch64-unknown-none-elf"
-// C-AARCH64-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
-// C-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
-
-// RUN: %clang -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=C-AARCH64-BAREMETAL-NOSYSROOT %s
-
-// C-AARCH64-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "aarch64-unknown-none-elf"
-// C-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL %s
-
-// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
-// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1/backward"
-// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/8.2.1"
-// CXX-AARCH64-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf -stdlib=libstdc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT %s
-
-// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/aarch64-none-elf"
-// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1/backward"
-// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/8.2.1"
-// CXX-AARCH64-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf 2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-LIBCXX %s
-
-// CXX-AARCH64-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_aarch64_gcc_tree/aarch64-none-elf"
-// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include/c++/v1"
-// CXX-AARCH64-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/aarch64-none-elf/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=aarch64-none-elf -stdlib=libc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX %s
-
-// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include/c++/v1"
-// CXX-AARCH64-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_aarch64_gcc_tree/lib/gcc/aarch64-none-elf/8.2.1/../../../../aarch64-none-elf/include"
diff --git a/clang/test/Driver/arm-gnutools.c b/clang/test/Driver/arm-gnutools.c
deleted file mode 100644
index 6e107f19dabc..000000000000
--- a/clang/test/Driver/arm-gnutools.c
+++ /dev/null
@@ -1,6 +0,0 @@
-// check that gnu assembler is invoked with arm baremetal as well
-
-// RUN: %clang --target=armv6m-none-eabi  --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -fno-integrated-as %s -### -c \
-// RUN: 2>&1 | FileCheck %s
-
-// CHECK: "{{.*}}as{{(.exe)?}}"
diff --git a/clang/test/Driver/arm-toolchain-extra.c b/clang/test/Driver/arm-toolchain-extra.c
deleted file mode 100644
index 67206818f211..000000000000
--- a/clang/test/Driver/arm-toolchain-extra.c
+++ /dev/null
@@ -1,30 +0,0 @@
-// A basic clang -cc1 command-line, and simple environment check.
-
-// The tests here are similar to those in arm-toolchain.c, however
-// these tests need to create symlinks to test directory trees in order to
-// set up the environment and therefore shell support is required.
-// XFAIL: target={{.*}}-fuchsia{{.*}}
-// REQUIRES: shell
-// UNSUPPORTED: system-windows
-
-// If there is no GCC install detected then the driver searches for executables
-// and runtime starting from the directory tree above the driver itself.
-// The test below checks that the driver correctly finds the linker and
-// runtime if and only if they exist.
-//
-// RUN: rm -rf %t
-// RUN: mkdir -p %t/arm-nogcc/bin
-// RUN: ln -s %clang %t/arm-nogcc/bin/clang
-// RUN: ln -s %S/Inputs/basic_arm_nogcc_tree/armv6m-none-eabi %t/arm-nogcc/armv6m-none-eabi
-// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    --gcc-toolchain=%t/arm-nogcc/invalid \
-// RUN:    --target=armv6m-none-eabi --rtlib=libgcc -fuse-ld=ld 2>&1 \
-// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
-
-// RUN: %t/arm-nogcc/bin/clang %s -### -no-canonical-prefixes \
-// RUN:    --sysroot=%t/arm-nogcc/bin/../armv6m-none-eabi \
-// RUN:    --target=armv6m-none-eabi --rtlib=libgcc -fuse-ld=ld 2>&1 \
-// RUN:    | FileCheck -check-prefix=C-ARM-BAREMETAL-NOGCC %s
-
-// C-ARM-BAREMETAL-NOGCC: "-internal-isystem" "{{.*}}/arm-nogcc/bin/../armv6m-none-eabi/include"
-
diff --git a/clang/test/Driver/arm-toolchain.c b/clang/test/Driver/arm-toolchain.c
deleted file mode 100644
index 56a0e0de7ba7..000000000000
--- a/clang/test/Driver/arm-toolchain.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// XFAIL: target={{.*}}-fuchsia{{.*}}
-// UNSUPPORTED: system-windows
-
-// RUN: %clang -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
-// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL %s
-
-// C-ARM-BAREMETAL: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
-// C-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
-// C-ARM-BAREMETAL: "-internal-isystem" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
-
-// RUN: %clang -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=C-ARM-BAREMETAL-NOSYSROOT %s
-
-// C-ARM-BAREMETAL-NOSYSROOT: "-cc1" "-triple" "thumbv6m-unknown-none-eabi"
-// C-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL %s
-
-// CXX-ARM-BAREMETAL: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
-// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
-// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1/backward"
-// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/8.2.1" 
-// CXX-ARM-BAREMETAL: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi -stdlib=libstdc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT %s
-
-// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/armv6m-none-eabi"
-// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1/backward"
-// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/8.2.1"
-// CXX-ARM-BAREMETAL-NOSYSROOT: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=%S/Inputs/basic_arm_gcc_tree/armv6m-none-eabi 2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-LIBCXX %s
-
-// CXX-ARM-BAREMETAL-LIBCXX: "-isysroot" "{{.*}}Inputs/basic_arm_gcc_tree/armv6m-none-eabi"
-// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include/c++/v1"
-// CXX-ARM-BAREMETAL-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/armv6m-none-eabi/include"
-
-// RUN: %clangxx -### %s -fuse-ld= \
-// RUN:   --target=armv6m-none-eabi -stdlib=libc++ --rtlib=libgcc \
-// RUN:   --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree \
-// RUN:   --sysroot=  2>&1 \
-// RUN:   | FileCheck -check-prefix=CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX %s
-
-// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include/c++/v1"
-// CXX-ARM-BAREMETAL-NOSYSROOT-LIBCXX: "-internal-isystem" "{{.*}}/Inputs/basic_arm_gcc_tree/lib/gcc/armv6m-none-eabi/8.2.1/../../../../armv6m-none-eabi/include
diff --git a/clang/test/Driver/baremetal.cpp b/clang/test/Driver/baremetal.cpp
index 2ac83402dda3..a80aa9b43711 100644
--- a/clang/test/Driver/baremetal.cpp
+++ b/clang/test/Driver/baremetal.cpp
@@ -196,22 +196,6 @@
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
 // CHECK-AARCH64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
 
-// RUN: %clang -no-canonical-prefixes %s -### --target=riscv32-unknown-elf 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-RISCV32-NO-HOST-INC %s
-// CHECK-RISCV32-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
-// CHECK-RISCV32-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
-// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
-// CHECK-RISCV32-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-
-// RUN: %clang -no-canonical-prefixes %s -### --target=riscv64-unknown-elf 2>&1 \
-// RUN:   | FileCheck --check-prefix=CHECK-RISCV64-NO-HOST-INC %s
-// CHECK-RISCV64-NO-HOST-INC: InstalledDir: [[INSTALLEDDIR:.+]]
-// CHECK-RISCV64-NO-HOST-INC: "-resource-dir" "[[RESOURCE:[^"]+]]"
-// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include{{[/\\]+}}c++{{[/\\]+}}v1"
-// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[RESOURCE]]{{[/\\]+}}include"
-// CHECK-RISCV64-NO-HOST-INC-SAME: "-internal-isystem" "[[INSTALLEDDIR]]{{[/\\]+}}..{{[/\\]+}}lib{{[/\\]+}}clang-runtimes{{[/\\]+[^"]*}}include"
-
 // RUN: %clang %s -### --target=riscv64-unknown-elf -o %t.out -L some/directory/user/asked/for \
 // RUN:     --sysroot=%S/Inputs/basic_riscv64_tree/riscv64-unknown-elf 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-RV64 %s
diff --git a/clang/test/Driver/check-no-multlib-warning.c b/clang/test/Driver/check-no-multlib-warning.c
deleted file mode 100644
index 9a0d7cee450a..000000000000
--- a/clang/test/Driver/check-no-multlib-warning.c
+++ /dev/null
@@ -1,10 +0,0 @@
-// UNSUPPORTED: system-windows
-
-
-// RUN: %clang --target=armv6m-none-eabi --gcc-toolchain=%S/Inputs/basic_arm_gcc_tree -### 2>&1 | FileCheck %s
-// RUN: %clang --target=aarch64-none-elf --gcc-toolchain=%S/Inputs/basic_aarch64_gcc_tree -### 2>&1 | FileCheck %s
-// RUN: %clang --target=riscv32-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv32_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
-// RUN: %clang --target=riscv64-unknown-elf --gcc-toolchain=%S/Inputs/basic_riscv64_tree -### 2>&1 | FileCheck --check-prefix=NOCHECK %s
-
-// CHECK: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
-// NOCHECK-NOT: warning: no multilib structure encoded for Arm, Aarch64 and PPC targets
-- 
cgit v1.2.3


From 1f34d68c4f086e7ea6ef9a529f9606476b38bbbb Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Wed, 18 Jun 2025 14:25:41 +0100
Subject: [Remarks] Remove yaml-strtab format (#144527)

Background: The yaml-strtab format looks just like the yaml format,
except that the values in the key/value pairs of the remarks are
deduplicated and replaced by indices into a string table (see removed
test cases for examples). The motivation behind this format was to
reduce size of the remarks files. However, it was quickly superseded by
the bitstream format.

Therefore, remove the yaml-strtab format, as it doesn't have a good
usecase anymore:
  - It isn't particularly efficient
  - It isn't human-readable
  - It isn't straightforward to parse in external tools that can't use the
remarks library. We don't even support it in opt-viewer.

llvm-remarkutil is also missing options to parse/convert yaml-strtab, so
the chance that anyone is actually using this format is low.
---
 llvm/docs/CommandGuide/llvm-opt-report.rst         |   1 -
 llvm/docs/Remarks.rst                              |  32 +---
 llvm/include/llvm/Remarks/RemarkFormat.h           |   2 +-
 llvm/include/llvm/Remarks/RemarkParser.h           |   5 -
 llvm/include/llvm/Remarks/YAMLRemarkSerializer.h   |  41 ------
 llvm/lib/Remarks/BitstreamRemarkParser.cpp         |   7 +-
 llvm/lib/Remarks/BitstreamRemarkParser.h           |   7 +-
 llvm/lib/Remarks/RemarkFormat.cpp                  |   4 +-
 llvm/lib/Remarks/RemarkLinker.cpp                  |   2 +-
 llvm/lib/Remarks/RemarkParser.cpp                  |  40 +----
 llvm/lib/Remarks/RemarkSerializer.cpp              |   5 -
 llvm/lib/Remarks/RemarkStreamer.cpp                |   4 +-
 llvm/lib/Remarks/YAMLRemarkParser.cpp              |  71 +--------
 llvm/lib/Remarks/YAMLRemarkParser.h                |  19 +--
 llvm/lib/Remarks/YAMLRemarkSerializer.cpp          | 103 +++----------
 llvm/test/CodeGen/X86/remarks-section.ll           |   8 -
 llvm/unittests/Remarks/RemarksLinkingTest.cpp      |  35 ++---
 llvm/unittests/Remarks/YAMLRemarksParsingTest.cpp  | 133 +----------------
 .../Remarks/YAMLRemarksSerializerTest.cpp          | 163 ---------------------
 19 files changed, 64 insertions(+), 618 deletions(-)

diff --git a/llvm/docs/CommandGuide/llvm-opt-report.rst b/llvm/docs/CommandGuide/llvm-opt-report.rst
index 4a666a4aa7af..ba10ba34578a 100644
--- a/llvm/docs/CommandGuide/llvm-opt-report.rst
+++ b/llvm/docs/CommandGuide/llvm-opt-report.rst
@@ -94,7 +94,6 @@ be sent to standard output.
  The Argument is one of the following:
 
  - yaml
- - yaml-strtab
  - bitstream
 
 .. option:: --no-demangle
diff --git a/llvm/docs/Remarks.rst b/llvm/docs/Remarks.rst
index b6cec12b326f..3be66e5adac9 100644
--- a/llvm/docs/Remarks.rst
+++ b/llvm/docs/Remarks.rst
@@ -112,7 +112,6 @@ following options:
       Supported formats:
 
       * :ref:`yaml <yamlremarks>` (default)
-      * :ref:`yaml-strtab <yamlstrtabremarks>`
       * :ref:`bitstream <bitstreamremarks>`
 
 ``Content configuration``
@@ -213,30 +212,6 @@ fields are required:
 * ``<arg-line>``
 * ``<arg-column>``
 
-.. _yamlstrtabremarks:
-
-YAML with a string table
-------------------------
-
-The YAML serialization supports the usage of a string table by using the
-``yaml-strtab`` format.
-
-This format replaces strings in the YAML output with integers representing the
-index in the string table that can be provided separately through metadata.
-
-The following entries can take advantage of the string table while respecting
-YAML rules:
-
-* ``<pass>``
-* ``<name>``
-* ``<function>``
-* ``<file>``
-* ``<value>``
-* ``<arg-file>``
-
-Currently, none of the tools in :ref:`the opt-viewer directory <optviewer>`
-support this format.
-
 .. _optviewer:
 
 YAML metadata
@@ -246,9 +221,9 @@ The metadata used together with the YAML format is:
 
 * a magic number: "REMARKS\\0"
 * the version number: a little-endian uint64_t
-* the total size of the string table (the size itself excluded):
-  little-endian uint64_t
-* a list of null-terminated strings
+* 8 zero bytes. This space was previously used to encode the size of a string
+  table. String table support for YAML remarks has been removed, use the
+  bitstream format instead.
 
 Optional:
 
@@ -584,7 +559,6 @@ Emitting remark diagnostics in the object file
 A section containing metadata on remark diagnostics will be emitted for the
 following formats:
 
-* ``yaml-strtab``
 * ``bitstream``
 
 This can be overridden by using the flag ``-remarks-section=<bool>``.
diff --git a/llvm/include/llvm/Remarks/RemarkFormat.h b/llvm/include/llvm/Remarks/RemarkFormat.h
index 64d08bcc9b8a..a39a013dcf90 100644
--- a/llvm/include/llvm/Remarks/RemarkFormat.h
+++ b/llvm/include/llvm/Remarks/RemarkFormat.h
@@ -23,7 +23,7 @@ namespace remarks {
 constexpr StringLiteral Magic("REMARKS");
 
 /// The format used for serializing/deserializing remarks.
-enum class Format { Unknown, YAML, YAMLStrTab, Bitstream };
+enum class Format { Unknown, YAML, Bitstream };
 
 /// Parse and validate a string for the remark format.
 LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
diff --git a/llvm/include/llvm/Remarks/RemarkParser.h b/llvm/include/llvm/Remarks/RemarkParser.h
index abb1fb86a87e..e3df74436348 100644
--- a/llvm/include/llvm/Remarks/RemarkParser.h
+++ b/llvm/include/llvm/Remarks/RemarkParser.h
@@ -80,13 +80,8 @@ struct ParsedStringTable {
 LLVM_ABI Expected<std::unique_ptr<RemarkParser>>
 createRemarkParser(Format ParserFormat, StringRef Buf);
 
-LLVM_ABI Expected<std::unique_ptr<RemarkParser>>
-createRemarkParser(Format ParserFormat, StringRef Buf,
-                   ParsedStringTable StrTab);
-
 LLVM_ABI Expected<std::unique_ptr<RemarkParser>> createRemarkParserFromMeta(
     Format ParserFormat, StringRef Buf,
-    std::optional<ParsedStringTable> StrTab = std::nullopt,
     std::optional<StringRef> ExternalFilePrependPath = std::nullopt);
 
 } // end namespace remarks
diff --git a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
index a2214c349e1c..d80464c0fe74 100644
--- a/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
+++ b/llvm/include/llvm/Remarks/YAMLRemarkSerializer.h
@@ -63,47 +63,6 @@ struct LLVM_ABI YAMLMetaSerializer : public MetaSerializer {
   void emit() override;
 };
 
-/// Serialize the remarks to YAML using a string table. An remark entry looks
-/// like the regular YAML remark but instead of string entries it's using
-/// numbers that map to an index in the string table.
-struct LLVM_ABI YAMLStrTabRemarkSerializer : public YAMLRemarkSerializer {
-  /// Wether we already emitted the metadata in standalone mode.
-  /// This should be set to true after the first invocation of `emit`.
-  bool DidEmitMeta = false;
-
-  YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode)
-      : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode) {
-    // We always need a string table for this type of serializer.
-    StrTab.emplace();
-  }
-  YAMLStrTabRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
-                             StringTable StrTab)
-      : YAMLRemarkSerializer(Format::YAMLStrTab, OS, Mode, std::move(StrTab)) {}
-
-  /// Override to emit the metadata if necessary.
-  void emit(const Remark &Remark) override;
-
-  std::unique_ptr<MetaSerializer> metaSerializer(
-      raw_ostream &OS,
-      std::optional<StringRef> ExternalFilename = std::nullopt) override;
-
-  static bool classof(const RemarkSerializer *S) {
-    return S->SerializerFormat == Format::YAMLStrTab;
-  }
-};
-
-struct LLVM_ABI YAMLStrTabMetaSerializer : public YAMLMetaSerializer {
-  /// The string table is part of the metadata.
-  const StringTable &StrTab;
-
-  YAMLStrTabMetaSerializer(raw_ostream &OS,
-                           std::optional<StringRef> ExternalFilename,
-                           const StringTable &StrTab)
-      : YAMLMetaSerializer(OS, ExternalFilename), StrTab(StrTab) {}
-
-  void emit() override;
-};
-
 } // end namespace remarks
 } // end namespace llvm
 
diff --git a/llvm/lib/Remarks/BitstreamRemarkParser.cpp b/llvm/lib/Remarks/BitstreamRemarkParser.cpp
index 6dd032f07e72..312886013598 100644
--- a/llvm/lib/Remarks/BitstreamRemarkParser.cpp
+++ b/llvm/lib/Remarks/BitstreamRemarkParser.cpp
@@ -308,8 +308,7 @@ static Error advanceToMetaBlock(BitstreamParserHelper &Helper) {
 
 Expected<std::unique_ptr<BitstreamRemarkParser>>
 remarks::createBitstreamParserFromMeta(
-    StringRef Buf, std::optional<ParsedStringTable> StrTab,
-    std::optional<StringRef> ExternalFilePrependPath) {
+    StringRef Buf, std::optional<StringRef> ExternalFilePrependPath) {
   BitstreamParserHelper Helper(Buf);
   Expected<std::array<char, 4>> MagicNumber = Helper.parseMagic();
   if (!MagicNumber)
@@ -319,9 +318,7 @@ remarks::createBitstreamParserFromMeta(
           StringRef(MagicNumber->data(), MagicNumber->size())))
     return std::move(E);
 
-  auto Parser =
-      StrTab ? std::make_unique<BitstreamRemarkParser>(Buf, std::move(*StrTab))
-             : std::make_unique<BitstreamRemarkParser>(Buf);
+  auto Parser = std::make_unique<BitstreamRemarkParser>(Buf);
 
   if (ExternalFilePrependPath)
     Parser->ExternalFilePrependPath = std::string(*ExternalFilePrependPath);
diff --git a/llvm/lib/Remarks/BitstreamRemarkParser.h b/llvm/lib/Remarks/BitstreamRemarkParser.h
index fc786fc57622..f6f79ef199f7 100644
--- a/llvm/lib/Remarks/BitstreamRemarkParser.h
+++ b/llvm/lib/Remarks/BitstreamRemarkParser.h
@@ -48,11 +48,6 @@ struct BitstreamRemarkParser : public RemarkParser {
   explicit BitstreamRemarkParser(StringRef Buf)
       : RemarkParser(Format::Bitstream), ParserHelper(Buf) {}
 
-  /// Create a parser that uses a pre-parsed string table.
-  BitstreamRemarkParser(StringRef Buf, ParsedStringTable StrTab)
-      : RemarkParser(Format::Bitstream), ParserHelper(Buf),
-        StrTab(std::move(StrTab)) {}
-
   Expected<std::unique_ptr<Remark>> next() override;
 
   static bool classof(const RemarkParser *P) {
@@ -77,7 +72,7 @@ private:
 };
 
 Expected<std::unique_ptr<BitstreamRemarkParser>> createBitstreamParserFromMeta(
-    StringRef Buf, std::optional<ParsedStringTable> StrTab = std::nullopt,
+    StringRef Buf,
     std::optional<StringRef> ExternalFilePrependPath = std::nullopt);
 
 } // end namespace remarks
diff --git a/llvm/lib/Remarks/RemarkFormat.cpp b/llvm/lib/Remarks/RemarkFormat.cpp
index 5006421a3c63..800f5bffe70d 100644
--- a/llvm/lib/Remarks/RemarkFormat.cpp
+++ b/llvm/lib/Remarks/RemarkFormat.cpp
@@ -20,7 +20,6 @@ using namespace llvm::remarks;
 Expected<Format> llvm::remarks::parseFormat(StringRef FormatStr) {
   auto Result = StringSwitch<Format>(FormatStr)
                     .Cases("", "yaml", Format::YAML)
-                    .Case("yaml-strtab", Format::YAMLStrTab)
                     .Case("bitstream", Format::Bitstream)
                     .Default(Format::Unknown);
 
@@ -36,7 +35,8 @@ Expected<Format> llvm::remarks::magicToFormat(StringRef MagicStr) {
   auto Result =
       StringSwitch<Format>(MagicStr)
           .StartsWith("--- ", Format::YAML) // This is only an assumption.
-          .StartsWith(remarks::Magic, Format::YAMLStrTab)
+          .StartsWith(remarks::Magic,
+                      Format::YAML) // Needed for remark meta section
           .StartsWith(remarks::ContainerMagic, Format::Bitstream)
           .Default(Format::Unknown);
 
diff --git a/llvm/lib/Remarks/RemarkLinker.cpp b/llvm/lib/Remarks/RemarkLinker.cpp
index b70b06d706bd..b8395aa135d8 100644
--- a/llvm/lib/Remarks/RemarkLinker.cpp
+++ b/llvm/lib/Remarks/RemarkLinker.cpp
@@ -76,7 +76,7 @@ Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
 
   Expected<std::unique_ptr<RemarkParser>> MaybeParser =
       createRemarkParserFromMeta(
-          *RemarkFormat, Buffer, /*StrTab=*/std::nullopt,
+          *RemarkFormat, Buffer,
           PrependPath ? std::optional<StringRef>(StringRef(*PrependPath))
                       : std::optional<StringRef>());
   if (!MaybeParser)
diff --git a/llvm/lib/Remarks/RemarkParser.cpp b/llvm/lib/Remarks/RemarkParser.cpp
index 7fccb94014b9..5c1690aaa0fe 100644
--- a/llvm/lib/Remarks/RemarkParser.cpp
+++ b/llvm/lib/Remarks/RemarkParser.cpp
@@ -53,10 +53,6 @@ llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) {
   switch (ParserFormat) {
   case Format::YAML:
     return std::make_unique<YAMLRemarkParser>(Buf);
-  case Format::YAMLStrTab:
-    return createStringError(
-        std::make_error_code(std::errc::invalid_argument),
-        "The YAML with string table format requires a parsed string table.");
   case Format::Bitstream:
     return std::make_unique<BitstreamRemarkParser>(Buf);
   case Format::Unknown:
@@ -66,38 +62,15 @@ llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) {
   llvm_unreachable("unhandled ParseFormat");
 }
 
-Expected<std::unique_ptr<RemarkParser>>
-llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf,
-                                  ParsedStringTable StrTab) {
-  switch (ParserFormat) {
-  case Format::YAML:
-    return createStringError(std::make_error_code(std::errc::invalid_argument),
-                             "The YAML format can't be used with a string "
-                             "table. Use yaml-strtab instead.");
-  case Format::YAMLStrTab:
-    return std::make_unique<YAMLStrTabRemarkParser>(Buf, std::move(StrTab));
-  case Format::Bitstream:
-    return std::make_unique<BitstreamRemarkParser>(Buf, std::move(StrTab));
-  case Format::Unknown:
-    return createStringError(std::make_error_code(std::errc::invalid_argument),
-                             "Unknown remark parser format.");
-  }
-  llvm_unreachable("unhandled ParseFormat");
-}
-
 Expected<std::unique_ptr<RemarkParser>>
 llvm::remarks::createRemarkParserFromMeta(
-    Format ParserFormat, StringRef Buf, std::optional<ParsedStringTable> StrTab,
+    Format ParserFormat, StringRef Buf,
     std::optional<StringRef> ExternalFilePrependPath) {
   switch (ParserFormat) {
-  // Depending on the metadata, the format can be either yaml or yaml-strtab,
-  // regardless of the input argument.
   case Format::YAML:
-  case Format::YAMLStrTab:
-    return createYAMLParserFromMeta(Buf, std::move(StrTab),
-                                    std::move(ExternalFilePrependPath));
+    return createYAMLParserFromMeta(Buf, std::move(ExternalFilePrependPath));
   case Format::Bitstream:
-    return createBitstreamParserFromMeta(Buf, std::move(StrTab),
+    return createBitstreamParserFromMeta(Buf,
                                          std::move(ExternalFilePrependPath));
   case Format::Unknown:
     return createStringError(std::make_error_code(std::errc::invalid_argument),
@@ -112,11 +85,8 @@ struct CParser {
   std::unique_ptr<RemarkParser> TheParser;
   std::optional<std::string> Err;
 
-  CParser(Format ParserFormat, StringRef Buf,
-          std::optional<ParsedStringTable> StrTab = std::nullopt)
-      : TheParser(cantFail(
-            StrTab ? createRemarkParser(ParserFormat, Buf, std::move(*StrTab))
-                   : createRemarkParser(ParserFormat, Buf))) {}
+  CParser(Format ParserFormat, StringRef Buf)
+      : TheParser(cantFail(createRemarkParser(ParserFormat, Buf))) {}
 
   void handleError(Error E) { Err.emplace(toString(std::move(E))); }
   bool hasError() const { return Err.has_value(); }
diff --git a/llvm/lib/Remarks/RemarkSerializer.cpp b/llvm/lib/Remarks/RemarkSerializer.cpp
index ab19c84bbadb..cc10b91f287a 100644
--- a/llvm/lib/Remarks/RemarkSerializer.cpp
+++ b/llvm/lib/Remarks/RemarkSerializer.cpp
@@ -26,8 +26,6 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                              "Unknown remark serializer format.");
   case Format::YAML:
     return std::make_unique<YAMLRemarkSerializer>(OS, Mode);
-  case Format::YAMLStrTab:
-    return std::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode);
   case Format::Bitstream:
     return std::make_unique<BitstreamRemarkSerializer>(OS, Mode);
   }
@@ -43,9 +41,6 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                              "Unknown remark serializer format.");
   case Format::YAML:
     return std::make_unique<YAMLRemarkSerializer>(OS, Mode, std::move(StrTab));
-  case Format::YAMLStrTab:
-    return std::make_unique<YAMLStrTabRemarkSerializer>(OS, Mode,
-                                                        std::move(StrTab));
   case Format::Bitstream:
     return std::make_unique<BitstreamRemarkSerializer>(OS, Mode,
                                                        std::move(StrTab));
diff --git a/llvm/lib/Remarks/RemarkStreamer.cpp b/llvm/lib/Remarks/RemarkStreamer.cpp
index 9f4676ce37ab..bb62c8b5c2fd 100644
--- a/llvm/lib/Remarks/RemarkStreamer.cpp
+++ b/llvm/lib/Remarks/RemarkStreamer.cpp
@@ -21,7 +21,7 @@ static cl::opt<cl::boolOrDefault> EnableRemarksSection(
     "remarks-section",
     cl::desc(
         "Emit a section containing remark diagnostics metadata. By default, "
-        "this is enabled for the following formats: yaml-strtab, bitstream."),
+        "this is enabled for the following formats: bitstream."),
     cl::init(cl::BOU_UNSET), cl::Hidden);
 
 RemarkStreamer::RemarkStreamer(
@@ -63,9 +63,7 @@ bool RemarkStreamer::needsSection() const {
 
   // Only some formats need a section:
   // * bitstream
-  // * yaml-strtab
   switch (RemarkSerializer->SerializerFormat) {
-  case remarks::Format::YAMLStrTab:
   case remarks::Format::Bitstream:
     return true;
   default:
diff --git a/llvm/lib/Remarks/YAMLRemarkParser.cpp b/llvm/lib/Remarks/YAMLRemarkParser.cpp
index a287ef574255..5ff42fe6b9a9 100644
--- a/llvm/lib/Remarks/YAMLRemarkParser.cpp
+++ b/llvm/lib/Remarks/YAMLRemarkParser.cpp
@@ -95,21 +95,8 @@ static Expected<uint64_t> parseStrTabSize(StringRef &Buf) {
   return StrTabSize;
 }
 
-static Expected<ParsedStringTable> parseStrTab(StringRef &Buf,
-                                               uint64_t StrTabSize) {
-  if (Buf.size() < StrTabSize)
-    return createStringError(std::errc::illegal_byte_sequence,
-                             "Expecting string table.");
-
-  // Attach the string table to the parser.
-  ParsedStringTable Result(StringRef(Buf.data(), StrTabSize));
-  Buf = Buf.drop_front(StrTabSize);
-  return Expected<ParsedStringTable>(std::move(Result));
-}
-
 Expected<std::unique_ptr<YAMLRemarkParser>> remarks::createYAMLParserFromMeta(
-    StringRef Buf, std::optional<ParsedStringTable> StrTab,
-    std::optional<StringRef> ExternalFilePrependPath) {
+    StringRef Buf, std::optional<StringRef> ExternalFilePrependPath) {
   // We now have a magic number. The metadata has to be correct.
   Expected<bool> isMeta = parseMagic(Buf);
   if (!isMeta)
@@ -125,15 +112,9 @@ Expected<std::unique_ptr<YAMLRemarkParser>> remarks::createYAMLParserFromMeta(
     if (!StrTabSize)
       return StrTabSize.takeError();
 
-    // If the size of string table is not 0, try to build one.
     if (*StrTabSize != 0) {
-      if (StrTab)
-        return createStringError(std::errc::illegal_byte_sequence,
-                                 "String table already provided.");
-      Expected<ParsedStringTable> MaybeStrTab = parseStrTab(Buf, *StrTabSize);
-      if (!MaybeStrTab)
-        return MaybeStrTab.takeError();
-      StrTab = std::move(*MaybeStrTab);
+      return createStringError(std::errc::illegal_byte_sequence,
+                               "String table unsupported for YAML format.");
     }
     // If it starts with "---", there is no external file.
     if (!Buf.starts_with("---")) {
@@ -157,21 +138,15 @@ Expected<std::unique_ptr<YAMLRemarkParser>> remarks::createYAMLParserFromMeta(
   }
 
   std::unique_ptr<YAMLRemarkParser> Result =
-      StrTab
-          ? std::make_unique<YAMLStrTabRemarkParser>(Buf, std::move(*StrTab))
-          : std::make_unique<YAMLRemarkParser>(Buf);
+      std::make_unique<YAMLRemarkParser>(Buf);
   if (SeparateBuf)
     Result->SeparateBuf = std::move(SeparateBuf);
   return std::move(Result);
 }
 
 YAMLRemarkParser::YAMLRemarkParser(StringRef Buf)
-    : YAMLRemarkParser(Buf, std::nullopt) {}
-
-YAMLRemarkParser::YAMLRemarkParser(StringRef Buf,
-                                   std::optional<ParsedStringTable> StrTab)
-    : RemarkParser{Format::YAML}, StrTab(std::move(StrTab)),
-      SM(setupSM(LastErrorMessage)), Stream(Buf, SM), YAMLIt(Stream.begin()) {}
+    : RemarkParser{Format::YAML}, SM(setupSM(LastErrorMessage)),
+      Stream(Buf, SM), YAMLIt(Stream.begin()) {}
 
 Error YAMLRemarkParser::error(StringRef Message, yaml::Node &Node) {
   return make_error<YAMLParseError>(Message, SM, Stream, Node);
@@ -208,8 +183,8 @@ YAMLRemarkParser::parseRemark(yaml::Document &RemarkEntry) {
   Expected<Type> T = parseType(*Root);
   if (!T)
     return T.takeError();
-  else
-    TheRemark.RemarkType = *T;
+
+  TheRemark.RemarkType = *T;
 
   // Then, parse the fields, one by one.
   for (yaml::KeyValueNode &RemarkField : *Root) {
@@ -428,33 +403,3 @@ Expected<std::unique_ptr<Remark>> YAMLRemarkParser::next() {
 
   return std::move(*MaybeResult);
 }
-
-Expected<StringRef> YAMLStrTabRemarkParser::parseStr(yaml::KeyValueNode &Node) {
-  auto *Value = dyn_cast<yaml::ScalarNode>(Node.getValue());
-  yaml::BlockScalarNode *ValueBlock;
-  StringRef Result;
-  if (!Value) {
-    // Try to parse the value as a block node.
-    ValueBlock = dyn_cast<yaml::BlockScalarNode>(Node.getValue());
-    if (!ValueBlock)
-      return error("expected a value of scalar type.", Node);
-    Result = ValueBlock->getValue();
-  } else
-    Result = Value->getRawValue();
-  // If we have a string table, parse it as an unsigned.
-  unsigned StrID = 0;
-  if (Expected<unsigned> MaybeStrID = parseUnsigned(Node))
-    StrID = *MaybeStrID;
-  else
-    return MaybeStrID.takeError();
-
-  if (Expected<StringRef> Str = (*StrTab)[StrID])
-    Result = *Str;
-  else
-    return Str.takeError();
-
-  Result.consume_front("\'");
-  Result.consume_back("\'");
-
-  return Result;
-}
diff --git a/llvm/lib/Remarks/YAMLRemarkParser.h b/llvm/lib/Remarks/YAMLRemarkParser.h
index 8ef72e16be74..9a30e9e295cb 100644
--- a/llvm/lib/Remarks/YAMLRemarkParser.h
+++ b/llvm/lib/Remarks/YAMLRemarkParser.h
@@ -46,8 +46,6 @@ private:
 
 /// Regular YAML to Remark parser.
 struct YAMLRemarkParser : public RemarkParser {
-  /// The string table used for parsing strings.
-  std::optional<ParsedStringTable> StrTab;
   /// Last error message that can come from the YAML parser diagnostics.
   /// We need this for catching errors in the constructor.
   std::string LastErrorMessage;
@@ -70,7 +68,6 @@ struct YAMLRemarkParser : public RemarkParser {
   }
 
 protected:
-  YAMLRemarkParser(StringRef Buf, std::optional<ParsedStringTable> StrTab);
   /// Create a YAMLParseError error from an existing error generated by the YAML
   /// parser.
   /// If there is no error, this returns Success.
@@ -93,22 +90,8 @@ protected:
   Expected<Argument> parseArg(yaml::Node &Node);
 };
 
-/// YAML with a string table to Remark parser.
-struct YAMLStrTabRemarkParser : public YAMLRemarkParser {
-  YAMLStrTabRemarkParser(StringRef Buf, ParsedStringTable StrTab)
-      : YAMLRemarkParser(Buf, std::move(StrTab)) {}
-
-  static bool classof(const RemarkParser *P) {
-    return P->ParserFormat == Format::YAMLStrTab;
-  }
-
-protected:
-  /// Parse one value to a string.
-  Expected<StringRef> parseStr(yaml::KeyValueNode &Node) override;
-};
-
 Expected<std::unique_ptr<YAMLRemarkParser>> createYAMLParserFromMeta(
-    StringRef Buf, std::optional<ParsedStringTable> StrTab = std::nullopt,
+    StringRef Buf,
     std::optional<StringRef> ExternalFilePrependPath = std::nullopt);
 
 } // end namespace remarks
diff --git a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
index 68285c3dde1b..846a72182d8f 100644
--- a/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
+++ b/llvm/lib/Remarks/YAMLRemarkSerializer.cpp
@@ -21,11 +21,10 @@ using namespace llvm::remarks;
 
 // Use the same keys whether we use a string table or not (respectively, T is an
 // unsigned or a StringRef).
-template <typename T>
-static void mapRemarkHeader(yaml::IO &io, T PassName, T RemarkName,
-                            std::optional<RemarkLocation> RL, T FunctionName,
-                            std::optional<uint64_t> Hotness,
-                            ArrayRef<Argument> Args) {
+static void
+mapRemarkHeader(yaml::IO &io, StringRef PassName, StringRef RemarkName,
+                std::optional<RemarkLocation> RL, StringRef FunctionName,
+                std::optional<uint64_t> Hotness, ArrayRef<Argument> Args) {
   io.mapRequired("Pass", PassName);
   io.mapRequired("Name", RemarkName);
   io.mapOptional("DebugLoc", RL);
@@ -58,19 +57,8 @@ template <> struct MappingTraits<remarks::Remark *> {
     else
       llvm_unreachable("Unknown remark type");
 
-    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
-            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
-      assert(Serializer->StrTab && "YAMLStrTabSerializer with no StrTab.");
-      StringTable &StrTab = *Serializer->StrTab;
-      unsigned PassID = StrTab.add(Remark->PassName).first;
-      unsigned NameID = StrTab.add(Remark->RemarkName).first;
-      unsigned FunctionID = StrTab.add(Remark->FunctionName).first;
-      mapRemarkHeader(io, PassID, NameID, Remark->Loc, FunctionID,
-                      Remark->Hotness, Remark->Args);
-    } else {
-      mapRemarkHeader(io, Remark->PassName, Remark->RemarkName, Remark->Loc,
-                      Remark->FunctionName, Remark->Hotness, Remark->Args);
-    }
+    mapRemarkHeader(io, Remark->PassName, Remark->RemarkName, Remark->Loc,
+                    Remark->FunctionName, Remark->Hotness, Remark->Args);
   }
 };
 
@@ -82,15 +70,7 @@ template <> struct MappingTraits<RemarkLocation> {
     unsigned Line = RL.SourceLine;
     unsigned Col = RL.SourceColumn;
 
-    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
-            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
-      assert(Serializer->StrTab && "YAMLStrTabSerializer with no StrTab.");
-      StringTable &StrTab = *Serializer->StrTab;
-      unsigned FileID = StrTab.add(File).first;
-      io.mapRequired("File", FileID);
-    } else {
-      io.mapRequired("File", File);
-    }
+    io.mapRequired("File", File);
 
     io.mapRequired("Line", Line);
     io.mapRequired("Column", Col);
@@ -136,13 +116,7 @@ template <> struct MappingTraits<Argument> {
   static void mapping(IO &io, Argument &A) {
     assert(io.outputting() && "input not yet implemented");
 
-    if (auto *Serializer = dyn_cast<YAMLStrTabRemarkSerializer>(
-            reinterpret_cast<RemarkSerializer *>(io.getContext()))) {
-      assert(Serializer->StrTab && "YAMLStrTabSerializer with no StrTab.");
-      StringTable &StrTab = *Serializer->StrTab;
-      auto ValueID = StrTab.add(A.Val).first;
-      io.mapRequired(A.Key.data(), ValueID);
-    } else if (StringRef(A.Val).count('\n') > 1) {
+    if (StringRef(A.Val).count('\n') > 1) {
       StringBlockVal S(A.Val);
       io.mapRequired(A.Key.data(), S);
     } else {
@@ -159,12 +133,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(Argument)
 
 YAMLRemarkSerializer::YAMLRemarkSerializer(raw_ostream &OS, SerializerMode Mode,
                                            std::optional<StringTable> StrTabIn)
-    : YAMLRemarkSerializer(Format::YAML, OS, Mode, std::move(StrTabIn)) {}
-
-YAMLRemarkSerializer::YAMLRemarkSerializer(Format SerializerFormat,
-                                           raw_ostream &OS, SerializerMode Mode,
-                                           std::optional<StringTable> StrTabIn)
-    : RemarkSerializer(SerializerFormat, OS, Mode),
+    : RemarkSerializer(Format::YAML, OS, Mode),
       YAMLOutput(OS, reinterpret_cast<void *>(this)) {
   StrTab = std::move(StrTabIn);
 }
@@ -172,7 +141,7 @@ YAMLRemarkSerializer::YAMLRemarkSerializer(Format SerializerFormat,
 void YAMLRemarkSerializer::emit(const Remark &Remark) {
   // Again, YAMLTraits expect a non-const object for inputting, but we're not
   // using that here.
-  auto R = const_cast<remarks::Remark *>(&Remark);
+  auto *R = const_cast<remarks::Remark *>(&Remark);
   YAMLOutput << R;
 }
 
@@ -181,27 +150,6 @@ std::unique_ptr<MetaSerializer> YAMLRemarkSerializer::metaSerializer(
   return std::make_unique<YAMLMetaSerializer>(OS, ExternalFilename);
 }
 
-void YAMLStrTabRemarkSerializer::emit(const Remark &Remark) {
-  // In standalone mode, for the serializer with a string table, emit the
-  // metadata first and set DidEmitMeta to avoid emitting it again.
-  if (Mode == SerializerMode::Standalone && !DidEmitMeta) {
-    std::unique_ptr<MetaSerializer> MetaSerializer =
-        metaSerializer(OS, /*ExternalFilename=*/std::nullopt);
-    MetaSerializer->emit();
-    DidEmitMeta = true;
-  }
-
-  // Then do the usual remark emission.
-  YAMLRemarkSerializer::emit(Remark);
-}
-
-std::unique_ptr<MetaSerializer> YAMLStrTabRemarkSerializer::metaSerializer(
-    raw_ostream &OS, std::optional<StringRef> ExternalFilename) {
-  assert(StrTab);
-  return std::make_unique<YAMLStrTabMetaSerializer>(OS, ExternalFilename,
-                                                    *StrTab);
-}
-
 static void emitMagic(raw_ostream &OS) {
   // Emit the magic number.
   OS << remarks::Magic;
@@ -216,20 +164,6 @@ static void emitVersion(raw_ostream &OS) {
   OS.write(Version.data(), Version.size());
 }
 
-static void emitStrTab(raw_ostream &OS,
-                       std::optional<const StringTable *> StrTab) {
-  // Emit the string table in the section.
-  uint64_t StrTabSize = StrTab ? (*StrTab)->SerializedSize : 0;
-  // Emit the total size of the string table (the size itself excluded):
-  // little-endian uint64_t.
-  // Note: even if no string table is used, emit 0.
-  std::array<char, 8> StrTabSizeBuf;
-  support::endian::write64le(StrTabSizeBuf.data(), StrTabSize);
-  OS.write(StrTabSizeBuf.data(), StrTabSizeBuf.size());
-  if (StrTab)
-    (*StrTab)->serialize(OS);
-}
-
 static void emitExternalFile(raw_ostream &OS, StringRef Filename) {
   // Emit the null-terminated absolute path to the remark file.
   SmallString<128> FilenameBuf = Filename;
@@ -242,15 +176,16 @@ static void emitExternalFile(raw_ostream &OS, StringRef Filename) {
 void YAMLMetaSerializer::emit() {
   emitMagic(OS);
   emitVersion(OS);
-  emitStrTab(OS, std::nullopt);
-  if (ExternalFilename)
-    emitExternalFile(OS, *ExternalFilename);
-}
 
-void YAMLStrTabMetaSerializer::emit() {
-  emitMagic(OS);
-  emitVersion(OS);
-  emitStrTab(OS, &StrTab);
+  // Emit StringTable with size 0. This is left over after removing StringTable
+  // support from the YAML format. For now, don't unnecessarily change how the
+  // the metadata is serialized. When changing the format, we should think about
+  // just reusing the bitstream remark meta for this.
+  uint64_t StrTabSize = 0;
+  std::array<char, 8> StrTabSizeBuf;
+  support::endian::write64le(StrTabSizeBuf.data(), StrTabSize);
+
+  OS.write(StrTabSizeBuf.data(), StrTabSizeBuf.size());
   if (ExternalFilename)
     emitExternalFile(OS, *ExternalFilename);
 }
diff --git a/llvm/test/CodeGen/X86/remarks-section.ll b/llvm/test/CodeGen/X86/remarks-section.ll
index dba20d428a69..e67c3579b759 100644
--- a/llvm/test/CodeGen/X86/remarks-section.ll
+++ b/llvm/test/CodeGen/X86/remarks-section.ll
@@ -1,8 +1,6 @@
 ; RUN: llc < %s -mtriple=x86_64-darwin -remarks-section -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN -DPATH=%/t.yaml %s
-; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml-strtab -remarks-section -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-STRTAB -DPATH=%/t.yaml %s
 
 ; RUN: llc < %s -mtriple=x86_64-darwin -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT %s
-; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml-strtab -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT-YAML-STRTAB %s
 ; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=bitstream -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-DEFAULT-BITSTREAM %s
 ; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=bitstream -remarks-section=false -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-BITSTREAM %s
 ; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml -remarks-section=true -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-YAML %s
@@ -10,15 +8,9 @@
 ; CHECK-DARWIN: .section __LLVM,__remarks,regular,debug
 ; CHECK-DARWIN-NEXT: .byte
 
-; CHECK-DARWIN-STRTAB: .section __LLVM,__remarks,regular,debug
-; CHECK-DARWIN-STRTAB-NEXT: .byte
-
 ; By default, the format is YAML which does not need a section.
 ; CHECK-DARWIN-DEFAULT-NOT: .section __LLVM,__remarks
 
-; yaml-strtab needs a section.
-; CHECK-DARWIN-DEFAULT-YAML-STRTAB: .section __LLVM,__remarks
-
 ; bitstream needs a section.
 ; CHECK-DARWIN-DEFAULT-BITSTREAM: .section __LLVM,__remarks
 
diff --git a/llvm/unittests/Remarks/RemarksLinkingTest.cpp b/llvm/unittests/Remarks/RemarksLinkingTest.cpp
index ff2aec669f2f..dcd598aaeb5c 100644
--- a/llvm/unittests/Remarks/RemarksLinkingTest.cpp
+++ b/llvm/unittests/Remarks/RemarksLinkingTest.cpp
@@ -207,22 +207,22 @@ TEST(Remarks, LinkingGoodStrTab) {
         "DebugLoc:        { File: file.c, Line: 3, Column: 12 }\n"
         "Function:        foo\n"
         "...\n",
-        remarks::Format::YAMLStrTab,
-        StringRef("REMARKS\0\0\0\0\0\0\0\0\0\x22\0\0\0\0\0\0\0"
-                  "inline\0NoDefinition\0foo\0file.c\0Ok\0"
-                  "--- !Passed\n"
-                  "Pass:            0\n"
-                  "Name:            4\n"
-                  "DebugLoc:        { File: 3, Line: 3, Column: 12 }\n"
-                  "Function:        2\n"
-                  "...\n"
-                  "--- !Missed\n"
-                  "Pass:            0\n"
-                  "Name:            1\n"
-                  "DebugLoc:        { File: 3, Line: 3, Column: 12 }\n"
-                  "Function:        2\n"
-                  "...\n",
-                  304));
+        remarks::Format::Bitstream,
+        "<BLOCKINFO_BLOCK/>\n"
+        "<Meta BlockID=8 NumWords=13 BlockCodeSize=3>\n"
+        "  <Container info codeid=1 abbrevid=4 op0=0 op1=2/>\n"
+        "  <Remark version codeid=2 abbrevid=5 op0=0/>\n"
+        "  <String table codeid=3 abbrevid=6/> blob data = "
+        "'inline\\x00NoDefinition\\x00foo\\x00file.c\\x00Ok\\x00'\n"
+        "</Meta>\n"
+        "<Remark BlockID=9 NumWords=4 BlockCodeSize=4>\n"
+        "  <Remark header codeid=5 abbrevid=4 op0=1 op1=4 op2=0 op3=2/>\n"
+        "  <Remark debug location codeid=6 abbrevid=5 op0=3 op1=3 op2=12/>\n"
+        "</Remark>\n"
+        "<Remark BlockID=9 NumWords=4 BlockCodeSize=4>\n"
+        "  <Remark header codeid=5 abbrevid=4 op0=2 op1=1 op2=0 op3=2/>\n"
+        "  <Remark debug location codeid=6 abbrevid=5 op0=3 op1=3 op2=12/>\n"
+        "</Remark>\n");
 }
 
 // Check that we propagate parsing errors.
@@ -241,11 +241,12 @@ TEST(Remarks, LinkingError) {
 
   {
     // Check that the prepend path is propagated and fails with the full path.
+    // Also ensures that the remark format is correctly auto-detected.
     RL.setExternalFilePrependPath("/baddir/");
     Error E = RL.link(
         StringRef("REMARKS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0badfile.opt.yaml",
                   40),
-        remarks::Format::YAMLStrTab);
+        /*RemarkFormat=*/std::nullopt);
     EXPECT_TRUE(static_cast<bool>(E));
     std::string ErrorMessage = toString(std::move(E));
     EXPECT_EQ(StringRef(ErrorMessage).lower(),
diff --git a/llvm/unittests/Remarks/YAMLRemarksParsingTest.cpp b/llvm/unittests/Remarks/YAMLRemarksParsingTest.cpp
index 3c740ddc8a55..824813aa5af7 100644
--- a/llvm/unittests/Remarks/YAMLRemarksParsingTest.cpp
+++ b/llvm/unittests/Remarks/YAMLRemarksParsingTest.cpp
@@ -77,7 +77,6 @@ void parseExpectErrorMeta(
 
   Expected<std::unique_ptr<remarks::RemarkParser>> MaybeParser =
       remarks::createRemarkParserFromMeta(remarks::Format::YAML, Buf,
-                                          /*StrTab=*/std::nullopt,
                                           std::move(ExternalFilePrependPath));
   handleAllErrors(MaybeParser.takeError(),
                   [&](const ErrorInfoBase &EIB) { EIB.log(Stream); });
@@ -558,124 +557,6 @@ TEST(YAMLRemarks, ContentsCAPI) {
   LLVMRemarkParserDispose(Parser);
 }
 
-TEST(YAMLRemarks, ContentsStrTab) {
-  StringRef Buf = "\n"
-                  "--- !Missed\n"
-                  "Pass: 0\n"
-                  "Name: 1\n"
-                  "DebugLoc: { File: 2, Line: 3, Column: 12 }\n"
-                  "Function: 3\n"
-                  "Hotness: 4\n"
-                  "Args:\n"
-                  "  - Callee: 5\n"
-                  "  - String: 7\n"
-                  "  - Caller: 3\n"
-                  "    DebugLoc: { File: 2, Line: 2, Column: 0 }\n"
-                  "  - String: 8\n"
-                  "\n";
-
-  StringRef StrTabBuf =
-      StringRef("inline\0NoDefinition\0file.c\0foo\0Callee\0bar\0String\0 "
-                "will not be inlined into \0 because its definition is "
-                "unavailable",
-                115);
-
-  remarks::ParsedStringTable StrTab(StrTabBuf);
-  Expected<std::unique_ptr<remarks::RemarkParser>> MaybeParser =
-      remarks::createRemarkParser(remarks::Format::YAMLStrTab, Buf,
-                                  std::move(StrTab));
-  EXPECT_FALSE(errorToBool(MaybeParser.takeError()));
-  EXPECT_TRUE(*MaybeParser != nullptr);
-
-  remarks::RemarkParser &Parser = **MaybeParser;
-  Expected<std::unique_ptr<remarks::Remark>> MaybeRemark = Parser.next();
-  EXPECT_FALSE(
-      errorToBool(MaybeRemark.takeError())); // Check for parsing errors.
-  EXPECT_TRUE(*MaybeRemark != nullptr);      // At least one remark.
-
-  const remarks::Remark &Remark = **MaybeRemark;
-  EXPECT_EQ(Remark.RemarkType, remarks::Type::Missed);
-  EXPECT_EQ(checkStr(Remark.PassName, 6), "inline");
-  EXPECT_EQ(checkStr(Remark.RemarkName, 12), "NoDefinition");
-  EXPECT_EQ(checkStr(Remark.FunctionName, 3), "foo");
-  EXPECT_TRUE(Remark.Loc);
-  const remarks::RemarkLocation &RL = *Remark.Loc;
-  EXPECT_EQ(checkStr(RL.SourceFilePath, 6), "file.c");
-  EXPECT_EQ(RL.SourceLine, 3U);
-  EXPECT_EQ(RL.SourceColumn, 12U);
-  EXPECT_TRUE(Remark.Hotness);
-  EXPECT_EQ(*Remark.Hotness, 4U);
-  EXPECT_EQ(Remark.Args.size(), 4U);
-
-  unsigned ArgID = 0;
-  for (const remarks::Argument &Arg : Remark.Args) {
-    switch (ArgID) {
-    case 0:
-      EXPECT_EQ(checkStr(Arg.Key, 6), "Callee");
-      EXPECT_EQ(checkStr(Arg.Val, 3), "bar");
-      EXPECT_FALSE(Arg.Loc);
-      break;
-    case 1:
-      EXPECT_EQ(checkStr(Arg.Key, 6), "String");
-      EXPECT_EQ(checkStr(Arg.Val, 26), " will not be inlined into ");
-      EXPECT_FALSE(Arg.Loc);
-      break;
-    case 2: {
-      EXPECT_EQ(checkStr(Arg.Key, 6), "Caller");
-      EXPECT_EQ(checkStr(Arg.Val, 3), "foo");
-      EXPECT_TRUE(Arg.Loc);
-      const remarks::RemarkLocation &RL = *Arg.Loc;
-      EXPECT_EQ(checkStr(RL.SourceFilePath, 6), "file.c");
-      EXPECT_EQ(RL.SourceLine, 2U);
-      EXPECT_EQ(RL.SourceColumn, 0U);
-      break;
-    }
-    case 3:
-      EXPECT_EQ(checkStr(Arg.Key, 6), "String");
-      EXPECT_EQ(checkStr(Arg.Val, 38),
-                " because its definition is unavailable");
-      EXPECT_FALSE(Arg.Loc);
-      break;
-    default:
-      break;
-    }
-    ++ArgID;
-  }
-
-  MaybeRemark = Parser.next();
-  Error E = MaybeRemark.takeError();
-  EXPECT_TRUE(E.isA<remarks::EndOfFileError>());
-  EXPECT_TRUE(errorToBool(std::move(E))); // Check for parsing errors.
-}
-
-TEST(YAMLRemarks, ParsingBadStringTableIndex) {
-  StringRef Buf = "\n"
-                  "--- !Missed\n"
-                  "Pass: 50\n"
-                  "\n";
-
-  StringRef StrTabBuf = StringRef("inline");
-
-  remarks::ParsedStringTable StrTab(StrTabBuf);
-  Expected<std::unique_ptr<remarks::RemarkParser>> MaybeParser =
-      remarks::createRemarkParser(remarks::Format::YAMLStrTab, Buf,
-                                  std::move(StrTab));
-  EXPECT_FALSE(errorToBool(MaybeParser.takeError()));
-  EXPECT_TRUE(*MaybeParser != nullptr);
-
-  remarks::RemarkParser &Parser = **MaybeParser;
-  Expected<std::unique_ptr<remarks::Remark>> MaybeRemark = Parser.next();
-  EXPECT_FALSE(MaybeRemark); // Expect an error here.
-
-  std::string ErrorStr;
-  raw_string_ostream Stream(ErrorStr);
-  handleAllErrors(MaybeRemark.takeError(),
-                  [&](const ErrorInfoBase &EIB) { EIB.log(Stream); });
-  EXPECT_TRUE(
-      StringRef(Stream.str())
-          .contains("String with index 50 is out of bounds (size = 1)."));
-}
-
 TEST(YAMLRemarks, ParsingGoodMeta) {
   // No metadata should also work.
   parseGoodMeta("--- !Missed\n"
@@ -692,17 +573,6 @@ TEST(YAMLRemarks, ParsingGoodMeta) {
                           "Name: NoDefinition\n"
                           "Function: foo\n",
                           82));
-
-  // Use the string table from the metadata.
-  parseGoodMeta(StringRef("REMARKS\0"
-                          "\0\0\0\0\0\0\0\0"
-                          "\x02\0\0\0\0\0\0\0"
-                          "a\0"
-                          "--- !Missed\n"
-                          "Pass: 0\n"
-                          "Name: 0\n"
-                          "Function: 0\n",
-                          66));
 }
 
 TEST(YAMLRemarks, ParsingBadMeta) {
@@ -727,7 +597,8 @@ TEST(YAMLRemarks, ParsingBadMeta) {
                                  "\0\0\0\0\0\0\0\0"
                                  "\x01\0\0\0\0\0\0\0",
                                  24),
-                       "Expecting string table.", CmpType::Equal);
+                       "String table unsupported for YAML format.",
+                       CmpType::Equal);
 
   parseExpectErrorMeta(StringRef("REMARKS\0"
                                  "\0\0\0\0\0\0\0\0"
diff --git a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
index 442c24b9fd95..7e994ac4d58b 100644
--- a/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
+++ b/llvm/unittests/Remarks/YAMLRemarksSerializerTest.cpp
@@ -131,78 +131,6 @@ TEST(YAMLRemarks, SerializerRemarkStandalone) {
                 "...\n"));
 }
 
-TEST(YAMLRemarks, SerializerRemarkStrTab) {
-  remarks::Remark R;
-  R.RemarkType = remarks::Type::Missed;
-  R.PassName = "pass";
-  R.RemarkName = "name";
-  R.FunctionName = "func";
-  R.Loc = remarks::RemarkLocation{"path", 3, 4};
-  R.Hotness = 5;
-  R.Args.emplace_back();
-  R.Args.back().Key = "key";
-  R.Args.back().Val = "value";
-  R.Args.emplace_back();
-  R.Args.back().Key = "keydebug";
-  R.Args.back().Val = "valuedebug";
-  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  check(remarks::Format::YAMLStrTab, R,
-        "--- !Missed\n"
-        "Pass:            0\n"
-        "Name:            1\n"
-        "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
-        "Function:        2\n"
-        "Hotness:         5\n"
-        "Args:\n"
-        "  - key:             4\n"
-        "  - keydebug:        5\n"
-        "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
-        "...\n",
-        StringRef("REMARKS\0"
-                  "\0\0\0\0\0\0\0\0"
-                  "\x2d\0\0\0\0\0\0\0"
-                  "pass\0name\0func\0path\0value\0valuedebug\0argpath"
-                  "\0" EXTERNALFILETESTPATH "\0",
-                  83));
-}
-
-TEST(YAMLRemarks, SerializerRemarkParsedStrTab) {
-  StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
-  remarks::Remark R;
-  R.RemarkType = remarks::Type::Missed;
-  R.PassName = "pass";
-  R.RemarkName = "name";
-  R.FunctionName = "func";
-  R.Loc = remarks::RemarkLocation{"path", 3, 4};
-  R.Hotness = 5;
-  R.Args.emplace_back();
-  R.Args.back().Key = "key";
-  R.Args.back().Val = "value";
-  R.Args.emplace_back();
-  R.Args.back().Key = "keydebug";
-  R.Args.back().Val = "valuedebug";
-  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  check(remarks::Format::YAMLStrTab, R,
-        "--- !Missed\n"
-        "Pass:            0\n"
-        "Name:            1\n"
-        "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
-        "Function:        2\n"
-        "Hotness:         5\n"
-        "Args:\n"
-        "  - key:             4\n"
-        "  - keydebug:        5\n"
-        "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
-        "...\n",
-        StringRef("REMARKS\0"
-                  "\0\0\0\0\0\0\0\0"
-                  "\x2d\0\0\0\0\0\0\0"
-                  "pass\0name\0func\0path\0value\0valuedebug\0argpath"
-                  "\0" EXTERNALFILETESTPATH "\0",
-                  83),
-        remarks::StringTable(remarks::ParsedStringTable(StrTab)));
-}
-
 TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneNoStrTab) {
   // Check that we don't use the string table even if it was provided.
   StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
@@ -237,94 +165,3 @@ TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneNoStrTab) {
                 "...\n"),
       std::move(PreFilledStrTab));
 }
-
-TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandalone) {
-  StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
-  remarks::ParsedStringTable ParsedStrTab(StrTab);
-  remarks::StringTable PreFilledStrTab(ParsedStrTab);
-  remarks::Remark R;
-  R.RemarkType = remarks::Type::Missed;
-  R.PassName = "pass";
-  R.RemarkName = "name";
-  R.FunctionName = "func";
-  R.Loc = remarks::RemarkLocation{"path", 3, 4};
-  R.Hotness = 5;
-  R.Args.emplace_back();
-  R.Args.back().Key = "key";
-  R.Args.back().Val = "value";
-  R.Args.emplace_back();
-  R.Args.back().Key = "keydebug";
-  R.Args.back().Val = "valuedebug";
-  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  checkStandalone(
-      remarks::Format::YAMLStrTab, R,
-      StringRef("REMARKS\0"
-                "\0\0\0\0\0\0\0\0"
-                "\x2d\0\0\0\0\0\0\0"
-                "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
-                "--- !Missed\n"
-                "Pass:            0\n"
-                "Name:            1\n"
-                "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
-                "Function:        2\n"
-                "Hotness:         5\n"
-                "Args:\n"
-                "  - key:             4\n"
-                "  - keydebug:        5\n"
-                "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
-                "...\n",
-                315),
-      std::move(PreFilledStrTab));
-}
-
-TEST(YAMLRemarks, SerializerRemarkParsedStrTabStandaloneMultipleRemarks) {
-  StringRef StrTab("pass\0name\0func\0path\0value\0valuedebug\0argpath\0", 45);
-  remarks::ParsedStringTable ParsedStrTab(StrTab);
-  remarks::StringTable PreFilledStrTab(ParsedStrTab);
-  SmallVector<remarks::Remark, 2> Rs;
-  remarks::Remark R;
-  R.RemarkType = remarks::Type::Missed;
-  R.PassName = "pass";
-  R.RemarkName = "name";
-  R.FunctionName = "func";
-  R.Loc = remarks::RemarkLocation{"path", 3, 4};
-  R.Hotness = 5;
-  R.Args.emplace_back();
-  R.Args.back().Key = "key";
-  R.Args.back().Val = "value";
-  R.Args.emplace_back();
-  R.Args.back().Key = "keydebug";
-  R.Args.back().Val = "valuedebug";
-  R.Args.back().Loc = remarks::RemarkLocation{"argpath", 6, 7};
-  Rs.emplace_back(R.clone());
-  Rs.emplace_back(std::move(R));
-  check(remarks::Format::YAMLStrTab, remarks::SerializerMode::Standalone, Rs,
-        StringRef("REMARKS\0"
-                  "\0\0\0\0\0\0\0\0"
-                  "\x2d\0\0\0\0\0\0\0"
-                  "pass\0name\0func\0path\0value\0valuedebug\0argpath\0"
-                  "--- !Missed\n"
-                  "Pass:            0\n"
-                  "Name:            1\n"
-                  "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
-                  "Function:        2\n"
-                  "Hotness:         5\n"
-                  "Args:\n"
-                  "  - key:             4\n"
-                  "  - keydebug:        5\n"
-                  "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
-                  "...\n"
-                  "--- !Missed\n"
-                  "Pass:            0\n"
-                  "Name:            1\n"
-                  "DebugLoc:        { File: 3, Line: 3, Column: 4 }\n"
-                  "Function:        2\n"
-                  "Hotness:         5\n"
-                  "Args:\n"
-                  "  - key:             4\n"
-                  "  - keydebug:        5\n"
-                  "    DebugLoc:        { File: 6, Line: 6, Column: 7 }\n"
-                  "...\n",
-                  561),
-        /*ExpectedMeta=*/std::nullopt, std::move(PreFilledStrTab));
-}
-- 
cgit v1.2.3


From 671caef379c603d2bcc428a00e3535b230162941 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles@arm.com>
Date: Wed, 18 Jun 2025 14:35:53 +0100
Subject: [Flang][OpenMP] Update relevant warnings to emit when OMP >= v5.2
 (#144492)

There has been a number of deprecation warnings that have been added to
Flang, however these features are only deprecated when the OpenMP
Version being used is 5.2 or later. Previously, flang did not consider
the version with the warnings so would always be emitted.

Flang now ensures warnings are emitted for the appropriate version of
OpenMP, and tests are updated to reflect this change.
---
 flang/lib/Semantics/resolve-directives.cpp           | 20 +++++++++++++-------
 flang/test/Semantics/OpenMP/allocate-align01.f90     |  2 +-
 flang/test/Semantics/OpenMP/allocate01.f90           |  2 +-
 flang/test/Semantics/OpenMP/allocate02.f90           |  2 --
 flang/test/Semantics/OpenMP/allocate03.f90           |  1 -
 flang/test/Semantics/OpenMP/allocate05.f90           |  2 --
 flang/test/Semantics/OpenMP/allocate06.f90           |  1 -
 flang/test/Semantics/OpenMP/allocate09.f90           |  5 -----
 flang/test/Semantics/OpenMP/clause-validity01.f90    |  9 ++++++++-
 flang/test/Semantics/OpenMP/deprecation.f90          |  2 +-
 flang/test/Semantics/OpenMP/flush02.f90              |  1 -
 flang/test/Semantics/OpenMP/nested-barrier.f90       |  2 --
 flang/test/Semantics/OpenMP/nested-master.f90        | 12 ------------
 flang/test/Semantics/OpenMP/nested-teams.f90         |  1 -
 flang/test/Semantics/OpenMP/ordered-simd.f90         |  4 ----
 flang/test/Semantics/OpenMP/parallel-master-goto.f90 |  1 -
 16 files changed, 24 insertions(+), 43 deletions(-)

diff --git a/flang/lib/Semantics/resolve-directives.cpp b/flang/lib/Semantics/resolve-directives.cpp
index 57db76e2160d..885c02e6ec74 100644
--- a/flang/lib/Semantics/resolve-directives.cpp
+++ b/flang/lib/Semantics/resolve-directives.cpp
@@ -835,8 +835,8 @@ private:
 
   void AddOmpRequiresToScope(Scope &, WithOmpDeclarative::RequiresFlags,
       std::optional<common::OmpMemoryOrderType>);
-  void IssueNonConformanceWarning(
-      llvm::omp::Directive D, parser::CharBlock source);
+  void IssueNonConformanceWarning(llvm::omp::Directive D,
+      parser::CharBlock source, unsigned EmitFromVersion);
 
   void CreateImplicitSymbols(const Symbol *symbol);
 
@@ -1668,7 +1668,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPBlockConstruct &x) {
   }
   if (beginDir.v == llvm::omp::Directive::OMPD_master ||
       beginDir.v == llvm::omp::Directive::OMPD_parallel_master)
-    IssueNonConformanceWarning(beginDir.v, beginDir.source);
+    IssueNonConformanceWarning(beginDir.v, beginDir.source, 52);
   ClearDataSharingAttributeObjects();
   ClearPrivateDataSharingAttributeObjects();
   ClearAllocateNames();
@@ -1791,7 +1791,7 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPLoopConstruct &x) {
       beginDir.v == llvm::omp::OMPD_parallel_master_taskloop ||
       beginDir.v == llvm::omp::OMPD_parallel_master_taskloop_simd ||
       beginDir.v == llvm::omp::Directive::OMPD_target_loop)
-    IssueNonConformanceWarning(beginDir.v, beginDir.source);
+    IssueNonConformanceWarning(beginDir.v, beginDir.source, 52);
   ClearDataSharingAttributeObjects();
   SetContextAssociatedLoopLevel(GetAssociatedLoopLevelFromClauses(clauseList));
 
@@ -2108,7 +2108,8 @@ bool OmpAttributeVisitor::Pre(const parser::OpenMPDispatchConstruct &x) {
 }
 
 bool OmpAttributeVisitor::Pre(const parser::OpenMPExecutableAllocate &x) {
-  IssueNonConformanceWarning(llvm::omp::Directive::OMPD_allocate, x.source);
+  IssueNonConformanceWarning(llvm::omp::Directive::OMPD_allocate, x.source, 52);
+
   PushContext(x.source, llvm::omp::Directive::OMPD_allocate);
   const auto &list{std::get<std::optional<parser::OmpObjectList>>(x.t)};
   if (list) {
@@ -3172,11 +3173,16 @@ void OmpAttributeVisitor::AddOmpRequiresToScope(Scope &scope,
   } while (!scopeIter->IsGlobal());
 }
 
-void OmpAttributeVisitor::IssueNonConformanceWarning(
-    llvm::omp::Directive D, parser::CharBlock source) {
+void OmpAttributeVisitor::IssueNonConformanceWarning(llvm::omp::Directive D,
+    parser::CharBlock source, unsigned EmitFromVersion) {
   std::string warnStr;
   llvm::raw_string_ostream warnStrOS(warnStr);
   unsigned version{context_.langOptions().OpenMPVersion};
+  // We only want to emit the warning when the version being used has the
+  // directive deprecated
+  if (version < EmitFromVersion) {
+    return;
+  }
   warnStrOS << "OpenMP directive "
             << parser::ToUpperCaseLetters(
                    llvm::omp::getOpenMPDirectiveName(D, version).str())
diff --git a/flang/test/Semantics/OpenMP/allocate-align01.f90 b/flang/test/Semantics/OpenMP/allocate-align01.f90
index 4974f5e18397..bc17d7047bbb 100644
--- a/flang/test/Semantics/OpenMP/allocate-align01.f90
+++ b/flang/test/Semantics/OpenMP/allocate-align01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=51
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=52
 ! OpenMP Version 5.2
 ! The allocate clause's allocator modifier must be of type allocator_handle
 ! and the align modifier must be constant, positive integer expression
diff --git a/flang/test/Semantics/OpenMP/allocate01.f90 b/flang/test/Semantics/OpenMP/allocate01.f90
index 8a680eee743e..b205b2c79d65 100644
--- a/flang/test/Semantics/OpenMP/allocate01.f90
+++ b/flang/test/Semantics/OpenMP/allocate01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags -fopenmp-version=52
 ! OpenMP Version 5.0
 ! 2.11.3 allocate Directive
 ! The allocate directive must appear in the same scope as the declarations of
diff --git a/flang/test/Semantics/OpenMP/allocate02.f90 b/flang/test/Semantics/OpenMP/allocate02.f90
index 80ef60b31e70..8f0579e810bb 100644
--- a/flang/test/Semantics/OpenMP/allocate02.f90
+++ b/flang/test/Semantics/OpenMP/allocate02.f90
@@ -16,11 +16,9 @@ use omp_lib
   !ERROR: At most one ALLOCATOR clause can appear on the ALLOCATE directive
   !$omp allocate(x, y) allocator(omp_default_mem_alloc) allocator(omp_default_mem_alloc)
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
       allocate ( darray(a, b) )
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: At most one ALLOCATOR clause can appear on the ALLOCATE directive
   !$omp allocate(darray) allocator(omp_default_mem_alloc) allocator(omp_default_mem_alloc)
       allocate ( darray(a, b) )
diff --git a/flang/test/Semantics/OpenMP/allocate03.f90 b/flang/test/Semantics/OpenMP/allocate03.f90
index b8c6b8e5dee7..e35115f3897c 100644
--- a/flang/test/Semantics/OpenMP/allocate03.f90
+++ b/flang/test/Semantics/OpenMP/allocate03.f90
@@ -18,7 +18,6 @@ use omp_lib
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the ALLOCATE directive
   !$omp allocate(my_var%array)
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: A variable that is part of another variable (as an array or structure element) cannot appear on the ALLOCATE directive
   !$omp allocate(darray, my_var%array) allocator(omp_default_mem_alloc)
     allocate ( darray(a, b) )
diff --git a/flang/test/Semantics/OpenMP/allocate05.f90 b/flang/test/Semantics/OpenMP/allocate05.f90
index 2c81c4dbc82c..a787e8bb32a4 100644
--- a/flang/test/Semantics/OpenMP/allocate05.f90
+++ b/flang/test/Semantics/OpenMP/allocate05.f90
@@ -13,13 +13,11 @@ use omp_lib
   real, dimension (:,:), allocatable :: darray
 
   !$omp target
-      !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
       !$omp allocate allocator(omp_default_mem_alloc)
           allocate ( darray(a, b) )
   !$omp end target
 
   !$omp target
-      !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
       !ERROR: ALLOCATE directives that appear in a TARGET region must specify an allocator clause
       !$omp allocate
           allocate ( darray(a, b) )
diff --git a/flang/test/Semantics/OpenMP/allocate06.f90 b/flang/test/Semantics/OpenMP/allocate06.f90
index 7196bcac2b9b..e14134cd0730 100644
--- a/flang/test/Semantics/OpenMP/allocate06.f90
+++ b/flang/test/Semantics/OpenMP/allocate06.f90
@@ -14,7 +14,6 @@ use omp_lib
   !ERROR: List items specified in the ALLOCATE directive must not have the ALLOCATABLE attribute unless the directive is associated with an ALLOCATE statement
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(darray) allocator(omp_default_mem_alloc)
     allocate(darray(a, b))
 
diff --git a/flang/test/Semantics/OpenMP/allocate09.f90 b/flang/test/Semantics/OpenMP/allocate09.f90
index 645e97a3a33f..0f93a340fe1e 100644
--- a/flang/test/Semantics/OpenMP/allocate09.f90
+++ b/flang/test/Semantics/OpenMP/allocate09.f90
@@ -12,28 +12,23 @@ use omp_lib
   integer, dimension(:), allocatable :: a, b, c, d, e, f, &
                                         g, h, i, j, k, l
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(a) allocator(omp_default_mem_alloc)
     allocate(a(1), b(2))
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(c, d) allocator(omp_default_mem_alloc)
     allocate(c(3), d(4))
 
   !$omp allocate(e) allocator(omp_default_mem_alloc)
   !$omp allocate(f, g) allocator(omp_default_mem_alloc)
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate
     allocate(e(5), f(6), g(7))
 
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !ERROR: Object 'i' in ALLOCATE directive not found in corresponding ALLOCATE statement
   !$omp allocate(h, i) allocator(omp_default_mem_alloc)
     allocate(h(8))
 
   !ERROR: Object 'j' in ALLOCATE directive not found in corresponding ALLOCATE statement
   !$omp allocate(j, k) allocator(omp_default_mem_alloc)
-  !WARNING: OpenMP directive ALLOCATE has been deprecated, please use ALLOCATORS instead.
   !$omp allocate(l) allocator(omp_default_mem_alloc)
     allocate(k(9), l(10))
 
diff --git a/flang/test/Semantics/OpenMP/clause-validity01.f90 b/flang/test/Semantics/OpenMP/clause-validity01.f90
index 5e0d91914c44..6989a183e83e 100644
--- a/flang/test/Semantics/OpenMP/clause-validity01.f90
+++ b/flang/test/Semantics/OpenMP/clause-validity01.f90
@@ -1,6 +1,6 @@
 ! REQUIRES: openmp_runtime
 
-! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags %openmp_module_flag -fopenmp-version=51
+! RUN: %python %S/../test_errors.py %s %flang_fc1 %openmp_flags %openmp_module_flag -fopenmp-version=52
 use omp_lib
 ! Check OpenMP clause validity for the following directives:
 !
@@ -502,6 +502,7 @@ use omp_lib
   !$omp taskyield
   !$omp barrier
   !$omp taskwait
+  !WARNING: SOURCE dependence type is deprecated in OpenMP v5.2
   !ERROR: The SINK and SOURCE dependence types can only be used with the ORDERED directive, used here in the TASKWAIT construct
   !$omp taskwait depend(source)
   ! !$omp taskwait depend(sink:i-1)
@@ -509,12 +510,18 @@ use omp_lib
   ! !$omp target update from(arrayA) to(arrayB)
   ! !$omp target exit data map(from:arrayA) map(delete:arrayB)
   !$omp flush (c)
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !$omp flush acq_rel
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !$omp flush release
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !$omp flush acquire
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !ERROR: If memory-order-clause is RELEASE, ACQUIRE, or ACQ_REL, list items must not be specified on the FLUSH directive
   !$omp flush release (c)
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !$omp flush seq_cst
+  !WARNING: The syntax "FLUSH clause (object, ...)" has been deprecated, use "FLUSH(object, ...) clause" instead
   !ERROR: RELAXED clause is not allowed on the FLUSH directive
   !$omp flush relaxed
 
diff --git a/flang/test/Semantics/OpenMP/deprecation.f90 b/flang/test/Semantics/OpenMP/deprecation.f90
index e04f43026bbc..df15c3bcc0b1 100644
--- a/flang/test/Semantics/OpenMP/deprecation.f90
+++ b/flang/test/Semantics/OpenMP/deprecation.f90
@@ -1,4 +1,4 @@
-! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -Werror
+! RUN: %python %S/../test_errors.py %s %flang_fc1 -fopenmp -Werror -fopenmp-version=52
 
 ! Check for deprecation of master directive and its combined/composite variants
 
diff --git a/flang/test/Semantics/OpenMP/flush02.f90 b/flang/test/Semantics/OpenMP/flush02.f90
index 615332c6cf31..a7b170d58db5 100644
--- a/flang/test/Semantics/OpenMP/flush02.f90
+++ b/flang/test/Semantics/OpenMP/flush02.f90
@@ -78,7 +78,6 @@ use omp_lib
 
   !$omp parallel num_threads(4)
     array = (/1, 2, 3, 4, 5, 6, 7, 8, 9, 10/)
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
       !$omp flush (array)
     !$omp end master
diff --git a/flang/test/Semantics/OpenMP/nested-barrier.f90 b/flang/test/Semantics/OpenMP/nested-barrier.f90
index 5f51363d59e5..8565a09a18cd 100644
--- a/flang/test/Semantics/OpenMP/nested-barrier.f90
+++ b/flang/test/Semantics/OpenMP/nested-barrier.f90
@@ -75,7 +75,6 @@ program omp_nest_barrier
   end do
   !$omp end critical
 
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   do i = 1, 10
     k = k + 1
@@ -108,7 +107,6 @@ program omp_nest_barrier
   end do
   !$omp end ordered
 
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   do i = 1, 10
     !ERROR: `DISTRIBUTE` region has to be strictly nested inside `TEAMS` region.
diff --git a/flang/test/Semantics/OpenMP/nested-master.f90 b/flang/test/Semantics/OpenMP/nested-master.f90
index d51e366eb584..7e4bb32bb7be 100644
--- a/flang/test/Semantics/OpenMP/nested-master.f90
+++ b/flang/test/Semantics/OpenMP/nested-master.f90
@@ -9,7 +9,6 @@ program omp_nest_master
   !$omp do
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -17,7 +16,6 @@ program omp_nest_master
   end do
 
   !$omp sections 
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     do i = 1, 10
@@ -27,7 +25,6 @@ program omp_nest_master
   !$omp end sections
 
   !$omp single 
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     do i = 1, 10
@@ -41,7 +38,6 @@ program omp_nest_master
   !$omp task
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -52,7 +48,6 @@ program omp_nest_master
   !$omp taskloop
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
     j = j -1
@@ -63,7 +58,6 @@ program omp_nest_master
   !$omp target parallel do simd
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: The only OpenMP constructs that can be encountered during execution of a 'SIMD' region are the `ATOMIC` construct, the `LOOP` construct, the `SIMD` construct, the `SCAN` construct and the `ORDERED` construct with the `SIMD` clause.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$omp master
@@ -75,7 +69,6 @@ program omp_nest_master
   !$omp critical
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
     j = j -1
     !$omp end master
@@ -85,7 +78,6 @@ program omp_nest_master
   !$omp ordered
   do i = 1, 10
     k = k + 1
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$omp master
     j = j -1
     !$omp end master
@@ -99,7 +91,6 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !$omp master
       j = j -1
       !$omp end master
@@ -116,7 +107,6 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !$omp master
       j = j -1
       !$omp end master
@@ -133,7 +123,6 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
       !$omp master
       j = j -1
@@ -151,7 +140,6 @@ program omp_nest_master
     !$omp distribute
     do k =1, 10
       print *, "hello"
-      !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
       !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
       !$omp master
       j = j -1
diff --git a/flang/test/Semantics/OpenMP/nested-teams.f90 b/flang/test/Semantics/OpenMP/nested-teams.f90
index 974172ee9717..3c193ee00b95 100644
--- a/flang/test/Semantics/OpenMP/nested-teams.f90
+++ b/flang/test/Semantics/OpenMP/nested-teams.f90
@@ -42,7 +42,6 @@ program main
   !$omp end teams
   end do
 
-  !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
   !$omp master
   !ERROR: TEAMS region can only be strictly nested within the implicit parallel region or TARGET region
   !$omp teams
diff --git a/flang/test/Semantics/OpenMP/ordered-simd.f90 b/flang/test/Semantics/OpenMP/ordered-simd.f90
index c90ffb3bd1c5..50560139ea24 100644
--- a/flang/test/Semantics/OpenMP/ordered-simd.f90
+++ b/flang/test/Semantics/OpenMP/ordered-simd.f90
@@ -95,7 +95,6 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP CRITICAL  
     C =  C - A * B
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$OMP MASTER
     DO I = 1,N
       !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
@@ -108,7 +107,6 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP ORDERED  
     C =  C - A * B
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !$OMP MASTER
     DO I = 1,N
       !ERROR: `ORDERED` region may not be closely nested inside of `CRITICAL`, `ORDERED`, explicit `TASK` or `TASKLOOP` region.
@@ -121,7 +119,6 @@ SUBROUTINE ORDERED_BAD(N)
 
   !$OMP TASK  
     C =  C - A * B
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$OMP MASTER
     DO I = 1,N
@@ -136,7 +133,6 @@ SUBROUTINE ORDERED_BAD(N)
   !$OMP TASKLOOP
   DO J= 1,N  
     C =  C - A * B
-    !WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
     !ERROR: `MASTER` region may not be closely nested inside of `WORKSHARING`, `LOOP`, `TASK`, `TASKLOOP`, or `ATOMIC` region.
     !$OMP MASTER
     DO I = 1,N
diff --git a/flang/test/Semantics/OpenMP/parallel-master-goto.f90 b/flang/test/Semantics/OpenMP/parallel-master-goto.f90
index 72c8002ab4c5..01d14aaa46d3 100644
--- a/flang/test/Semantics/OpenMP/parallel-master-goto.f90
+++ b/flang/test/Semantics/OpenMP/parallel-master-goto.f90
@@ -7,7 +7,6 @@ do i = 1, 2
 !ERROR: invalid branch leaving an OpenMP structured block
   goto 10
 end do
-!WARNING: OpenMP directive MASTER has been deprecated, please use MASKED instead.
 !$omp master
 10 print *, i
 !$omp end master
-- 
cgit v1.2.3


From fda6b751f1b1356e65816f85fbc5b98e78337940 Mon Sep 17 00:00:00 2001
From: Eric Fiselier <eric@efcs.ca>
Date: Wed, 18 Jun 2025 09:30:18 -0400
Subject: Fix libc++ restarter job.

A while ago, the test workflow was updated with a new preemption regex,
however it was only applied to the test job, and not the job
that's actually restarting the failed libc++ test runs.

This fix should correct the issue and get the restarter working
again.
---
 .github/workflows/libcxx-restart-preempted-jobs.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/libcxx-restart-preempted-jobs.yaml b/.github/workflows/libcxx-restart-preempted-jobs.yaml
index 7b341d7f22e4..9706f0459922 100644
--- a/.github/workflows/libcxx-restart-preempted-jobs.yaml
+++ b/.github/workflows/libcxx-restart-preempted-jobs.yaml
@@ -33,7 +33,7 @@ jobs:
         with:
           script: |
             const failure_regex = /Process completed with exit code 1./
-            const preemption_regex = /The runner has received a shutdown signal/
+            const preemption_regex = /(The runner has received a shutdown signal)|(The operation was canceled)/
 
             const wf_run = context.payload.workflow_run
             core.notice(`Running on "${wf_run.display_title}" by @${wf_run.actor.login} (event: ${wf_run.event})\nWorkflow run URL: ${wf_run.html_url}`)
-- 
cgit v1.2.3


From bdac9580f3bc341ccbeeb743ecca656756f5aaec Mon Sep 17 00:00:00 2001
From: Mircea Trofin <mtrofin@google.com>
Date: Wed, 18 Jun 2025 06:40:06 -0700
Subject: [nfc][jt] Drop `std::optional` pointers (#144548)

The `std::optional` didn't add any semantics that couldn't be modeled with the pointers being `nullptr`.
---
 llvm/include/llvm/Transforms/Scalar/JumpThreading.h |  7 +++----
 llvm/lib/Transforms/Scalar/JumpThreading.cpp        | 14 +++++++-------
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
index 75b5cf2371fd..a03a38466b27 100644
--- a/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
+++ b/llvm/include/llvm/Transforms/Scalar/JumpThreading.h
@@ -85,8 +85,8 @@ class JumpThreadingPass : public PassInfoMixin<JumpThreadingPass> {
   LazyValueInfo *LVI = nullptr;
   AAResults *AA = nullptr;
   std::unique_ptr<DomTreeUpdater> DTU;
-  std::optional<BlockFrequencyInfo *> BFI;
-  std::optional<BranchProbabilityInfo *> BPI;
+  BlockFrequencyInfo *BFI = nullptr;
+  BranchProbabilityInfo *BPI = nullptr;
   bool ChangedSinceLastAnalysisUpdate = false;
   bool HasGuards = false;
 #ifndef LLVM_ENABLE_ABI_BREAKING_CHECKS
@@ -110,8 +110,7 @@ public:
                         TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
                         LazyValueInfo *LVI, AAResults *AA,
                         std::unique_ptr<DomTreeUpdater> DTU,
-                        std::optional<BlockFrequencyInfo *> BFI,
-                        std::optional<BranchProbabilityInfo *> BPI);
+                        BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI);
 
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 37b85bf9de81..b5dbef13289a 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -249,7 +249,7 @@ PreservedAnalyses JumpThreadingPass::run(Function &F,
       runImpl(F, &AM, &TLI, &TTI, &LVI, &AA,
               std::make_unique<DomTreeUpdater>(
                   &DT, nullptr, DomTreeUpdater::UpdateStrategy::Lazy),
-              std::nullopt, std::nullopt);
+              nullptr, nullptr);
 
   if (!Changed)
     return PreservedAnalyses::all();
@@ -283,8 +283,8 @@ bool JumpThreadingPass::runImpl(Function &F_, FunctionAnalysisManager *FAM_,
                                 TargetTransformInfo *TTI_, LazyValueInfo *LVI_,
                                 AliasAnalysis *AA_,
                                 std::unique_ptr<DomTreeUpdater> DTU_,
-                                std::optional<BlockFrequencyInfo *> BFI_,
-                                std::optional<BranchProbabilityInfo *> BPI_) {
+                                BlockFrequencyInfo *BFI_,
+                                BranchProbabilityInfo *BPI_) {
   LLVM_DEBUG(dbgs() << "Jump threading on function '" << F_.getName() << "'\n");
   F = &F_;
   FAM = FAM_;
@@ -3215,7 +3215,7 @@ BranchProbabilityInfo *JumpThreadingPass::getBPI() {
     assert(FAM && "Can't create BPI without FunctionAnalysisManager");
     BPI = FAM->getCachedResult<BranchProbabilityAnalysis>(*F);
   }
-  return *BPI;
+  return BPI;
 }
 
 BlockFrequencyInfo *JumpThreadingPass::getBFI() {
@@ -3223,7 +3223,7 @@ BlockFrequencyInfo *JumpThreadingPass::getBFI() {
     assert(FAM && "Can't create BFI without FunctionAnalysisManager");
     BFI = FAM->getCachedResult<BlockFrequencyAnalysis>(*F);
   }
-  return *BFI;
+  return BFI;
 }
 
 // Important note on validity of BPI/BFI. JumpThreading tries to preserve
@@ -3237,7 +3237,7 @@ BranchProbabilityInfo *JumpThreadingPass::getOrCreateBPI(bool Force) {
   if (Force)
     BPI = runExternalAnalysis<BranchProbabilityAnalysis>();
 
-  return *BPI;
+  return BPI;
 }
 
 BlockFrequencyInfo *JumpThreadingPass::getOrCreateBFI(bool Force) {
@@ -3248,5 +3248,5 @@ BlockFrequencyInfo *JumpThreadingPass::getOrCreateBFI(bool Force) {
   if (Force)
     BFI = runExternalAnalysis<BlockFrequencyAnalysis>();
 
-  return *BFI;
+  return BFI;
 }
-- 
cgit v1.2.3


From c5613dc8635000bc0e8396b8156d5639195776ab Mon Sep 17 00:00:00 2001
From: lorenzo chelini <l.chelini@icloud.com>
Date: Wed, 18 Jun 2025 15:49:00 +0200
Subject: [MLIR] Mark LLVM::FMAOp as legal (#144671)

Mark LLVM::FMAOp as legal in configureGpuToNVVMConversionLegality, since
we can handle intrinsic lowering in the NVPTX backend and emit
fma.rn.f32.
---
 mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp |  8 ++++----
 mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir        |  2 +-
 mlir/test/Integration/GPU/CUDA/dump-ptx.mlir           | 14 +++++++++++++-
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
index 958d0d085fce..cef250232daf 100644
--- a/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
+++ b/mlir/lib/Conversion/GPUToNVVM/LowerGpuOpsToNVVMOps.cpp
@@ -429,10 +429,10 @@ void mlir::configureGpuToNVVMConversionLegality(ConversionTarget &target) {
   target.addLegalDialect<::mlir::NVVM::NVVMDialect>();
   target.addIllegalDialect<gpu::GPUDialect>();
   target.addIllegalOp<LLVM::CopySignOp, LLVM::CosOp, LLVM::ExpOp, LLVM::Exp2Op,
-                      LLVM::FAbsOp, LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FMAOp,
-                      LLVM::FRemOp, LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op,
-                      LLVM::PowOp, LLVM::RoundEvenOp, LLVM::RoundOp,
-                      LLVM::SinOp, LLVM::SqrtOp>();
+                      LLVM::FAbsOp, LLVM::FCeilOp, LLVM::FFloorOp, LLVM::FRemOp,
+                      LLVM::LogOp, LLVM::Log10Op, LLVM::Log2Op, LLVM::PowOp,
+                      LLVM::RoundEvenOp, LLVM::RoundOp, LLVM::SinOp,
+                      LLVM::SqrtOp>();
 
   // TODO: Remove once we support replacing non-root ops.
   target.addLegalOp<gpu::YieldOp, gpu::GPUModuleOp>();
diff --git a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
index 6d4555e815b6..ef06af3ad316 100644
--- a/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
+++ b/mlir/test/Conversion/GPUToNVVM/gpu-to-nvvm.mlir
@@ -1027,7 +1027,7 @@ module attributes {transform.with_named_sequence} {
       legal_ops = ["func.func", "gpu.module", "gpu.yield"],
       illegal_dialects = ["gpu"],
       illegal_ops = ["llvm.copysign", "llvm.cos", "llvm.exp", "llvm.exp2", "llvm.fabs", "llvm.fceil",
-                    "llvm.ffloor", "llvm.fma", "llvm.frem", "llvm.log", "llvm.log10", "llvm.log2", "llvm.pow",
+                    "llvm.ffloor", "llvm.frem", "llvm.log", "llvm.log10", "llvm.log2", "llvm.pow",
                     "llvm.roundeven", "llvm.round", "llvm.sin", "llvm.sqrt"],
       partial_conversion
     } : !transform.any_op
diff --git a/mlir/test/Integration/GPU/CUDA/dump-ptx.mlir b/mlir/test/Integration/GPU/CUDA/dump-ptx.mlir
index 0cc5d8645bb3..27ec1ec435fe 100644
--- a/mlir/test/Integration/GPU/CUDA/dump-ptx.mlir
+++ b/mlir/test/Integration/GPU/CUDA/dump-ptx.mlir
@@ -2,7 +2,7 @@
 // RUN:  | mlir-opt -gpu-lower-to-nvvm-pipeline -debug-only=serialize-to-isa \
 // RUN:  2>&1 | FileCheck %s
 
-// CHECK: Generated by LLVM NVPTX Back-End
+// CHECK-LABEL: Generated by LLVM NVPTX Back-End
 // CHECK: .visible .func kernel_a()
 // CHECK: ret;
 gpu.module @bar {
@@ -11,3 +11,15 @@ gpu.module @bar {
     llvm.return
   }
 }
+
+// CHECK-LABEL: Generated by LLVM NVPTX Back-End
+// CHECK: .visible .func  ({{.+}}) fma(
+// CHECK: fma.rn.f32
+
+gpu.module @foo {
+  llvm.func @fma(%arg0: f32, %arg1: f32) -> f32
+    attributes { gpu.kernel } {
+    %res = llvm.intr.fma (%arg0, %arg1, %arg1) : (f32, f32, f32) -> f32
+    llvm.return %res : f32
+  }
+}
-- 
cgit v1.2.3


From 1d6f1029f7e8cf5468309078da3e85201844b625 Mon Sep 17 00:00:00 2001
From: Sergei Lebedev <185856+superbobry@users.noreply.github.com>
Date: Wed, 18 Jun 2025 14:53:20 +0100
Subject: [mlir] [python] Fixed the return type of
 `MemRefType.get_strides_and_offset` (#144523)

Previously, the return type for `offset` was `list[int]`, which clearly
is not right.
---
 mlir/python/mlir/_mlir_libs/_mlir/ir.pyi | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
index 6c5f91d757cd..70bca3c75d84 100644
--- a/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
+++ b/mlir/python/mlir/_mlir_libs/_mlir/ir.pyi
@@ -2119,7 +2119,7 @@ class MemRefType(ShapedType):
         """
     @property
     def typeid(self) -> TypeID: ...
-    def get_strides_and_offset(self) -> tuple[list[int], list[int]]:
+    def get_strides_and_offset(self) -> tuple[list[int], int]:
         """
         The strides and offset of the MemRef type.
         """
-- 
cgit v1.2.3


From 9db7502d229b48817521429c2a5d3fb84543fdf9 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Wed, 18 Jun 2025 15:55:06 +0200
Subject: [libc++] Move __has_iterator_typedefs to the up-to-C++17
 implementation of iterator_traits (#144265)

`__has_iterator_typedefs` is only used in the up-to-C++17 implementation
of `type_traits`. To make that clearer the struct is moved into that
code block.
---
 libcxx/include/__iterator/iterator_traits.h | 34 ++++++++++++++---------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/libcxx/include/__iterator/iterator_traits.h b/libcxx/include/__iterator/iterator_traits.h
index 221d36614db0..f727e8ff36df 100644
--- a/libcxx/include/__iterator/iterator_traits.h
+++ b/libcxx/include/__iterator/iterator_traits.h
@@ -71,23 +71,6 @@ struct random_access_iterator_tag : public bidirectional_iterator_tag {};
 struct contiguous_iterator_tag : public random_access_iterator_tag {};
 #endif
 
-template <class _Tp>
-struct __has_iterator_typedefs {
-private:
-  template <class _Up>
-  static false_type __test(...);
-  template <class _Up>
-  static true_type
-  __test(__void_t<typename _Up::iterator_category>* = nullptr,
-         __void_t<typename _Up::difference_type>*   = nullptr,
-         __void_t<typename _Up::value_type>*        = nullptr,
-         __void_t<typename _Up::reference>*         = nullptr,
-         __void_t<typename _Up::pointer>*           = nullptr);
-
-public:
-  static const bool value = decltype(__test<_Tp>(nullptr, nullptr, nullptr, nullptr, nullptr))::value;
-};
-
 #if _LIBCPP_STD_VER >= 20
 
 // The `cpp17-*-iterator` exposition-only concepts have very similar names to the `Cpp17*Iterator` named requirements
@@ -322,6 +305,23 @@ struct __iterator_traits<_Iter, true>
                               is_convertible<typename _Iter::iterator_category, input_iterator_tag>::value ||
                                   is_convertible<typename _Iter::iterator_category, output_iterator_tag>::value > {};
 
+template <class _Tp>
+struct __has_iterator_typedefs {
+private:
+  template <class _Up>
+  static false_type __test(...);
+  template <class _Up>
+  static true_type
+  __test(__void_t<typename _Up::iterator_category>* = nullptr,
+         __void_t<typename _Up::difference_type>*   = nullptr,
+         __void_t<typename _Up::value_type>*        = nullptr,
+         __void_t<typename _Up::reference>*         = nullptr,
+         __void_t<typename _Up::pointer>*           = nullptr);
+
+public:
+  static const bool value = decltype(__test<_Tp>(nullptr, nullptr, nullptr, nullptr, nullptr))::value;
+};
+
 // iterator_traits<Iterator> will only have the nested types if Iterator::iterator_category
 //    exists.  Else iterator_traits<Iterator> will be an empty class.  This is a
 //    conforming extension which allows some programs to compile and behave as
-- 
cgit v1.2.3


From 40d2f392106f43a60eea79f433b47a5ce44fc4a4 Mon Sep 17 00:00:00 2001
From: Akira Hatanaka <ahatanak@gmail.com>
Date: Wed, 18 Jun 2025 07:08:32 -0700
Subject: [Sema][ObjC] Loosen restrictions on reinterpret_cast involving
 indirect ARC-managed pointers (#144458)

Allow using reinterpret_cast for conversions between indirect ARC
pointers and other pointer types.

rdar://152905399
---
 clang/docs/ReleaseNotes.rst                  |  3 +++
 clang/include/clang/Sema/SemaObjC.h          |  3 ++-
 clang/lib/Sema/SemaCast.cpp                  | 11 +++++++----
 clang/lib/Sema/SemaExprObjC.cpp              | 12 ++++++++----
 clang/test/SemaObjCXX/arc-type-conversion.mm | 23 +++++++++++++++++++++--
 5 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 12816eed2e8b..18234188101f 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -327,6 +327,9 @@ Non-comprehensive list of changes in this release
   ``__reference_constructs_from_temporary`` should be used instead. (#GH44056)
 - Added `__builtin_get_vtable_pointer` to directly load the primary vtable pointer from a
   polymorphic object.
+- Clang no longer rejects reinterpret_cast conversions between indirect
+  ARC-managed pointers and other pointer types. The prior behavior was overly
+  strict and inconsistent with the ARC specification.
 
 New Compiler Flags
 ------------------
diff --git a/clang/include/clang/Sema/SemaObjC.h b/clang/include/clang/Sema/SemaObjC.h
index b629c6d29140..ed08ff0acf89 100644
--- a/clang/include/clang/Sema/SemaObjC.h
+++ b/clang/include/clang/Sema/SemaObjC.h
@@ -812,7 +812,8 @@ public:
                                           CheckedConversionKind CCK,
                                           bool Diagnose = true,
                                           bool DiagnoseCFAudited = false,
-                                          BinaryOperatorKind Opc = BO_PtrMemD);
+                                          BinaryOperatorKind Opc = BO_PtrMemD,
+                                          bool IsReinterpretCast = false);
 
   Expr *stripARCUnbridgedCast(Expr *e);
   void diagnoseARCUnbridgedCast(Expr *e);
diff --git a/clang/lib/Sema/SemaCast.cpp b/clang/lib/Sema/SemaCast.cpp
index 14e16bc39eb3..e15a43c11651 100644
--- a/clang/lib/Sema/SemaCast.cpp
+++ b/clang/lib/Sema/SemaCast.cpp
@@ -161,12 +161,14 @@ namespace {
       Self.CheckCastAlign(SrcExpr.get(), DestType, OpRange);
     }
 
-    void checkObjCConversion(CheckedConversionKind CCK) {
+    void checkObjCConversion(CheckedConversionKind CCK,
+                             bool IsReinterpretCast = false) {
       assert(Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers());
 
       Expr *src = SrcExpr.get();
-      if (Self.ObjC().CheckObjCConversion(OpRange, DestType, src, CCK) ==
-          SemaObjC::ACR_unbridged)
+      if (Self.ObjC().CheckObjCConversion(
+              OpRange, DestType, src, CCK, true, false, BO_PtrMemD,
+              IsReinterpretCast) == SemaObjC::ACR_unbridged)
         IsARCUnbridgedCast = true;
       SrcExpr = src;
     }
@@ -1263,7 +1265,8 @@ void CastOperation::CheckReinterpretCast() {
 
   if (isValidCast(tcr)) {
     if (Self.getLangOpts().allowsNonTrivialObjCLifetimeQualifiers())
-      checkObjCConversion(CheckedConversionKind::OtherCast);
+      checkObjCConversion(CheckedConversionKind::OtherCast,
+                          /*IsReinterpretCast=*/true);
     DiagnoseReinterpretUpDownCast(Self, SrcExpr.get(), DestType, OpRange);
 
     if (unsigned DiagID = checkCastFunctionType(Self, SrcExpr, DestType))
diff --git a/clang/lib/Sema/SemaExprObjC.cpp b/clang/lib/Sema/SemaExprObjC.cpp
index 3505d9f38d23..395f2f340dbd 100644
--- a/clang/lib/Sema/SemaExprObjC.cpp
+++ b/clang/lib/Sema/SemaExprObjC.cpp
@@ -4390,7 +4390,7 @@ SemaObjC::ARCConversionResult
 SemaObjC::CheckObjCConversion(SourceRange castRange, QualType castType,
                               Expr *&castExpr, CheckedConversionKind CCK,
                               bool Diagnose, bool DiagnoseCFAudited,
-                              BinaryOperatorKind Opc) {
+                              BinaryOperatorKind Opc, bool IsReinterpretCast) {
   ASTContext &Context = getASTContext();
   QualType castExprType = castExpr->getType();
 
@@ -4450,13 +4450,17 @@ SemaObjC::CheckObjCConversion(SourceRange castRange, QualType castType,
   // must be explicit.
   // Allow conversions between pointers to lifetime types and coreFoundation
   // pointers too, but only when the conversions are explicit.
+  // Allow conversions requested with a reinterpret_cast that converts an
+  // expression of type T* to type U*.
   if (exprACTC == ACTC_indirectRetainable &&
       (castACTC == ACTC_voidPtr ||
-       (castACTC == ACTC_coreFoundation && SemaRef.isCast(CCK))))
+       (castACTC == ACTC_coreFoundation && SemaRef.isCast(CCK)) ||
+       (IsReinterpretCast && effCastType->isAnyPointerType())))
     return ACR_okay;
   if (castACTC == ACTC_indirectRetainable &&
-      (exprACTC == ACTC_voidPtr || exprACTC == ACTC_coreFoundation) &&
-      SemaRef.isCast(CCK))
+      (((exprACTC == ACTC_voidPtr || exprACTC == ACTC_coreFoundation) &&
+        SemaRef.isCast(CCK)) ||
+       (IsReinterpretCast && castExprType->isAnyPointerType())))
     return ACR_okay;
 
   switch (ARCCastChecker(Context, exprACTC, castACTC, false).Visit(castExpr)) {
diff --git a/clang/test/SemaObjCXX/arc-type-conversion.mm b/clang/test/SemaObjCXX/arc-type-conversion.mm
index 64cfd02ec18c..0d281bf3e5c4 100644
--- a/clang/test/SemaObjCXX/arc-type-conversion.mm
+++ b/clang/test/SemaObjCXX/arc-type-conversion.mm
@@ -1,5 +1,8 @@
 // RUN: %clang_cc1 -fobjc-runtime-has-weak -fsyntax-only -fobjc-arc -verify -fblocks %s
 
+@class NSString;
+typedef unsigned __INTPTR_TYPE__ uintptr_t;
+
 void * cvt(id arg) // expected-note{{candidate function not viable: cannot convert argument of incomplete type 'void *' to '__strong id'}}
 {
   void* voidp_val;
@@ -72,6 +75,24 @@ void test_reinterpret_cast(__strong id *sip, __weak id *wip,
   (void)reinterpret_cast<__weak id *>(cwip); // expected-error{{reinterpret_cast from '__weak id const *' to '__weak id *' casts away qualifiers}}
   (void)reinterpret_cast<__weak id *>(csip); // expected-error{{reinterpret_cast from '__strong id const *' to '__weak id *' casts away qualifiers}}
   (void)reinterpret_cast<__strong id *>(cwip); // expected-error{{reinterpret_cast from '__weak id const *' to '__strong id *' casts away qualifiers}}
+
+  auto *ul = reinterpret_cast<unsigned long *>(sip);
+  (void)reinterpret_cast<__strong id *>(ul);
+  auto *wp = reinterpret_cast<__weak NSString *>(sip);
+  (void)reinterpret_cast<__strong id *>(wp);
+  (void)reinterpret_cast<unsigned long *>(csip); // expected-error {{reinterpret_cast from '__strong id const *' to 'unsigned long *' casts away qualifiers}}
+  (void)reinterpret_cast<const unsigned long *>(csip);
+  const unsigned long *cul = nullptr;
+  (void)reinterpret_cast<__strong id *>(cul); // expected-error {{reinterpret_cast from 'const unsigned long *' to '__strong id *' casts away qualifiers}}
+  (void)reinterpret_cast<const __strong id *>(cul);
+  volatile __strong id *vsip = nullptr;
+  (void)reinterpret_cast<unsigned long *>(vsip); // expected-error {{reinterpret_cast from '__strong id volatile *' to 'unsigned long *' casts away qualifiers}}
+  (void)reinterpret_cast<volatile unsigned long *>(vsip);
+  volatile unsigned long *vul = nullptr;
+  (void)reinterpret_cast<__strong id *>(vul); // expected-error {{reinterpret_cast from 'volatile unsigned long *' to '__strong id *' casts away qualifiers}}
+  (void)reinterpret_cast<volatile __strong id *>(vul);
+  auto uip = reinterpret_cast<uintptr_t>(sip);
+  (void)reinterpret_cast<__strong id *>(uip); // expected-error {{to '__strong id *' is disallowed with ARC}}
 }
 
 void test_cstyle_cast(__strong id *sip, __weak id *wip, 
@@ -194,8 +215,6 @@ typedef void (^Block)();
 typedef void (^Block_strong)() __strong;
 typedef void (^Block_autoreleasing)() __autoreleasing;
 
-@class NSString;
-
 void ownership_transfer_in_cast(void *vp, Block *pblk) {
   __strong NSString **sip2 = static_cast<NSString **>(static_cast<__strong id *>(vp));
   __strong NSString **&si2pref = static_cast<NSString **&>(sip2);
-- 
cgit v1.2.3


From ee070d08163ac09842d9bf0c1315f311df39faf1 Mon Sep 17 00:00:00 2001
From: Andrei Golubev <andrey.golubev@intel.com>
Date: Wed, 18 Jun 2025 16:18:12 +0200
Subject: [mlir][bufferization] Support custom types (1/N) (#142986)

Following the addition of TensorLike and BufferLike type interfaces (see
00eaff3e9c897c263a879416d0f151d7ca7eeaff), introduce minimal changes
required to bufferize a custom tensor operation into a custom buffer
operation.

To achieve this, new interface methods are added to TensorLike type
interface that abstract away the differences between existing (tensor ->
memref) and custom conversions.

The scope of the changes is intentionally limited (for example,
BufferizableOpInterface is untouched) in order to first understand the
basics and reach consensus design-wise.

---
Notable changes:
* mlir::bufferization::getBufferType() returns BufferLikeType (instead
of BaseMemRefType)
* ToTensorOp / ToBufferOp operate on TensorLikeType / BufferLikeType.
Operation argument "memref" renamed to "buffer"
* ToTensorOp's tensor type inferring builder is dropped (users now need
to provide the tensor type explicitly)
---
 .../Bufferization/IR/BufferizableOpInterface.h     | 18 +++++-
 .../Dialect/Bufferization/IR/BufferizationOps.td   | 59 ++++++++++----------
 .../Bufferization/IR/BufferizationTypeInterfaces.h |  7 +++
 .../IR/BufferizationTypeInterfaces.td              | 26 ++++++++-
 .../Bufferization/IR/UnstructuredControlFlow.h     |  5 +-
 .../Transforms/BufferizableOpInterfaceImpl.cpp     | 14 +++--
 .../Bufferization/IR/BufferizableOpInterface.cpp   | 65 +++++++++++++---------
 .../Bufferization/IR/BufferizationDialect.cpp      | 32 ++++++++++-
 .../Dialect/Bufferization/IR/BufferizationOps.cpp  | 26 +++++----
 .../IR/BufferizationTypeInterfaces.cpp             | 21 +++++++
 mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt   |  1 +
 .../Dialect/Bufferization/Transforms/Bufferize.cpp |  8 +--
 .../Transforms/FuncBufferizableOpInterfaceImpl.cpp |  8 +--
 .../Transforms/ConvertToDestinationStyle.cpp       |  9 ++-
 .../SCF/Transforms/BufferizableOpInterfaceImpl.cpp | 51 +++++++++--------
 .../Transforms/BufferizableOpInterfaceImpl.cpp     |  2 +-
 .../SparseTensor/Transforms/SparseGPUCodegen.cpp   | 15 +++--
 .../Transforms/SparseTensorCodegen.cpp             |  3 +-
 .../Transforms/SparseTensorConversion.cpp          |  4 +-
 .../SparseTensor/Transforms/Utils/CodegenUtils.cpp |  4 +-
 .../Transforms/BufferizableOpInterfaceImpl.cpp     | 14 +++--
 .../Transforms/one-shot-bufferize.mlir             | 21 ++++++-
 mlir/test/lib/Dialect/Test/TestOpDefs.cpp          | 23 ++++++++
 mlir/test/lib/Dialect/Test/TestOps.h               |  1 +
 mlir/test/lib/Dialect/Test/TestOps.td              | 58 ++++++++++++++++++-
 mlir/test/lib/Dialect/Test/TestTypeDefs.td         |  9 +++
 mlir/test/lib/Dialect/Test/TestTypes.cpp           | 20 +++++++
 27 files changed, 389 insertions(+), 135 deletions(-)
 create mode 100644 mlir/lib/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp

diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
index 2fb795f16ae2..c1529a36465a 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h
@@ -17,6 +17,7 @@
 #include <optional>
 
 #include "mlir/Dialect/Bufferization/IR/BufferizationEnums.h.inc"
+#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h"
 
 namespace mlir {
 class OpBuilder;
@@ -615,7 +616,7 @@ FailureOr<Value> getBuffer(RewriterBase &rewriter, Value value,
 /// IR, this function can be used.
 ///
 /// This function is a wrapper around BufferizableOpInterface::getBufferType.
-FailureOr<BaseMemRefType> getBufferType(Value value,
+FailureOr<BufferLikeType> getBufferType(Value value,
                                         const BufferizationOptions &options,
                                         const BufferizationState &state);
 
@@ -629,7 +630,7 @@ FailureOr<BaseMemRefType> getBufferType(Value value,
 /// IR, this function can be used.
 ///
 /// This function is a wrapper around `BufferizableOpInterface::getBufferType`.
-FailureOr<BaseMemRefType> getBufferType(Value value,
+FailureOr<BufferLikeType> getBufferType(Value value,
                                         const BufferizationOptions &options,
                                         const BufferizationState &state,
                                         SmallVector<Value> &invocationStack);
@@ -739,6 +740,19 @@ AliasingValueList unknownGetAliasingValues(OpOperand &opOperand);
 /// This is the default implementation of
 /// BufferizableOpInterface::hasTensorSemantics
 bool defaultHasTensorSemantics(Operation *op);
+
+/// This is a helper function used when buffer type is guaranteed to be memref.
+/// It performs two actions: failure state checking and an explicit llvm::cast<>
+/// from the buffer-like type interface to a BaseMemRefType. This allows easier
+/// management of differences in C++ types at the API boundaries. Valid buffer
+/// type is casted to the memref type. Otherwise, the failure state is
+/// propagated i.e. asMemRefType(mlir::failure()) returns mlir::failure().
+FailureOr<BaseMemRefType> asMemRefType(FailureOr<BufferLikeType> bufferType);
+
+/// This function is a free-standing helper that relies on
+/// bufferization::TensorLikeTypeInterface to verify the types in tensor and
+/// buffer worlds match.
+bool typesMatchAfterBufferization(Operation &op, Value tensor, Value buffer);
 } // namespace detail
 
 } // namespace bufferization
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
index 6051aea84997..32c53ea9c494 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationOps.td
@@ -12,6 +12,7 @@
 include "mlir/Dialect/Bufferization/IR/AllocationOpInterface.td"
 include "mlir/Dialect/Bufferization/IR/BufferViewFlowOpInterface.td"
 include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td"
+include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td"
 include "mlir/Dialect/Bufferization/IR/BufferizationBase.td"
 include "mlir/Interfaces/DestinationStyleOpInterface.td"
 include "mlir/Interfaces/InferTypeOpInterface.td"
@@ -386,20 +387,31 @@ def Bufferization_DeallocTensorOp : Bufferization_Op<"dealloc_tensor",
 // ToTensorOp
 //===----------------------------------------------------------------------===//
 
+class Bufferization_TensorAndBufferMatch<string tensor, string buffer> : PredOpTrait<
+  "specified tensor and buffer types match",
+  CPred<
+    "::mlir::bufferization::detail::typesMatchAfterBufferization("
+        "$_op, $" # tensor # ", $" # buffer #")"
+  >
+>;
+
 def Bufferization_ToTensorOp : Bufferization_Op<"to_tensor", [
     BufferizableOpInterface,
     SameOperandsAndResultShape,
     SameOperandsAndResultElementType,
-    AllElementTypesMatch<["memref", "result"]>
+    Bufferization_TensorAndBufferMatch<"result", "buffer">
   ]> {
-  let summary = "create a tensor from a `memref`";
+  let summary = "create a buffer-like type from a tensor-like type";
   let description = [{
-    An operation that creates a tensor from a `memref`. The result value is a
-    tensor whose shape and element type match the memref operand.
+    An operation that creates a tensor from a buffer. The result value is a
+    tensor-like type that must match the corresponding buffer-like operand as
+    per TensorLikeType::verifyCompatibleBufferType(). For builtins (TensorType
+    and BaseMemRefType), this means that shapes and element types match between
+    the tensor and the buffer.
 
     The opposite of this op is `to_buffer`. Together, these two ops are
     useful for source/target materializations when doing type conversions
-    involving tensors and memrefs.
+    involving tensors and buffers.
 
     Example:
 
@@ -441,19 +453,16 @@ def Bufferization_ToTensorOp : Bufferization_Op<"to_tensor", [
     away. However, such IR is no longer bufferizable with One-Shot Bufferize.
   }];
 
-  let arguments = (ins Arg<AnyRankedOrUnrankedMemRef,
+  let arguments = (ins Arg<Bufferization_BufferLikeTypeInterface,
                            "the reference to load from",
-                           [MemReadAt<0, FullEffect>]>:$memref,
+                           [MemReadAt<0, FullEffect>]>:$buffer,
                        UnitAttr:$restrict, UnitAttr:$writable);
-  let results = (outs AnyTensor:$result);
+  let results = (outs Bufferization_TensorLikeTypeInterface:$result);
 
   let extraClassDeclaration = [{
     /// The result of a to_tensor is always a tensor.
-    TensorType getType() {
-      Type resultType = getResult().getType();
-      if (::llvm::isa<TensorType>(resultType))
-        return ::llvm::cast<TensorType>(resultType);
-      return {};
+    ::mlir::bufferization::TensorLikeType getType() {
+      return getResult().getType();
     }
 
     //===------------------------------------------------------------------===//
@@ -472,22 +481,15 @@ def Bufferization_ToTensorOp : Bufferization_Op<"to_tensor", [
     FailureOr<BaseMemRefType> getBufferType(
         Value value, const BufferizationOptions &options,
         const BufferizationState &state, SmallVector<Value> &invocationStack) {
-      return ::llvm::cast<BaseMemRefType>(getMemref().getType());
+      return ::llvm::cast<BaseMemRefType>(getBuffer().getType());
     }
   }];
 
   let assemblyFormat = [{
-    $memref (`restrict` $restrict^)? (`writable` $writable^)? attr-dict
-      `:` type($memref) `to` type($result)
+    $buffer (`restrict` $restrict^)? (`writable` $writable^)? attr-dict
+      `:` type($buffer) `to` type($result)
   }];
 
-  let builders = [
-    OpBuilder<(ins "Value":$memref, CArg<"bool", "false">:$restrict, CArg<"bool", "false">:$writeable), [{
-      auto rtt = memref::getTensorTypeFromMemRefType(memref.getType());
-      build($_builder, $_state, rtt, memref, restrict, writeable);
-    }]>
-  ];
-
   let hasCanonicalizer = 1;
   let hasFolder = 1;
 }
@@ -502,10 +504,9 @@ def Bufferization_ToBufferOp : Bufferization_Op<"to_buffer", [
     SameOperandsAndResultShape,
     SameOperandsAndResultElementType,
     Pure,
-    AllShapesMatch<["memref", "tensor"]>,
-    AllElementTypesMatch<["memref", "tensor"]>
+    Bufferization_TensorAndBufferMatch<"tensor", "buffer">
   ]> {
-  let summary = "cast a tensor to memref";
+  let summary = "cast a tensor-like type to buffer-like type";
   let description = [{
     An operation that returns the future buffer of a `tensor`.
 
@@ -523,8 +524,8 @@ def Bufferization_ToBufferOp : Bufferization_Op<"to_buffer", [
     the returned buffer) will not be written to.
   }];
 
-  let arguments = (ins AnyTensor:$tensor, UnitAttr:$read_only);
-  let results = (outs AnyRankedOrUnrankedMemRef:$memref);
+  let arguments = (ins Bufferization_TensorLikeTypeInterface:$tensor, UnitAttr:$read_only);
+  let results = (outs Bufferization_BufferLikeTypeInterface:$buffer);
 
   let extraClassDeclaration = [{
     //===------------------------------------------------------------------===//
@@ -559,7 +560,7 @@ def Bufferization_ToBufferOp : Bufferization_Op<"to_buffer", [
   }];
 
   let assemblyFormat = [{
-    $tensor (`read_only` $read_only^)? attr-dict `:` type($tensor) `to` type($memref)
+    $tensor (`read_only` $read_only^)? attr-dict `:` type($tensor) `to` type($buffer)
   }];
 
   let hasFolder = 1;
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
index 5faa1479ee54..cbb6054fcf88 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h
@@ -13,8 +13,15 @@
 // Bufferization Type Interfaces
 //===----------------------------------------------------------------------===//
 
+#include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Types.h"
 
+namespace mlir::bufferization {
+struct BufferizationOptions;
+class BufferizationState;
+class BufferLikeType;
+} // namespace mlir::bufferization
+
 #include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h.inc"
 
 #endif // MLIR_DIALECT_BUFFERIZATION_IR_BUFFERIZATIONTYPEINTERFACES_H_
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td
index f19224a29564..fb6fc4f5ad96 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.td
@@ -21,10 +21,30 @@ def Bufferization_TensorLikeTypeInterface
   let description = [{
     Indicates that this type is a tensor type (similarly to a MLIR builtin
     tensor) for bufferization purposes.
-
-    The interface currently has no methods as it is used by types to opt into
-    being supported by the bufferization procedures.
   }];
+
+  let methods = [
+    InterfaceMethod<[{
+        Returns a BufferLike type for this TensorLike type.
+      }],
+      /*retTy=*/"::mlir::FailureOr<::mlir::bufferization::BufferLikeType>",
+      /*methodName=*/"getBufferType",
+      /*args=*/(ins
+        "const ::mlir::bufferization::BufferizationOptions &":$options,
+        "::llvm::function_ref<::mlir::InFlightDiagnostic()>":$emitError
+      )
+    >,
+    InterfaceMethod<[{
+        Returns whether a BufferLike type is compatible to this TensorLike type.
+        The BufferLike type is assumed to be created by getBufferType().
+      }],
+      /*retTy=*/"::mlir::LogicalResult",
+      /*methodName=*/"verifyCompatibleBufferType",
+      /*args=*/(ins
+        "::mlir::bufferization::BufferLikeType":$bufferType,
+        "::llvm::function_ref<::mlir::InFlightDiagnostic()>":$emitError)
+    >
+  ];
 }
 
 def Bufferization_BufferLikeTypeInterface
diff --git a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
index a441b8b66659..f56c10555f02 100644
--- a/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
+++ b/mlir/include/mlir/Dialect/Bufferization/IR/UnstructuredControlFlow.h
@@ -65,12 +65,13 @@ struct OpWithUnstructuredControlFlowBufferizableOpInterfaceExternalModel
         // The operand was already bufferized. Take its type directly.
         callerType = memrefType;
       } else {
-        FailureOr<BaseMemRefType> maybeCallerType =
+        FailureOr<BufferLikeType> maybeCallerType =
             bufferization::getBufferType(opOperand->get(), options, state,
                                          invocationStack);
         if (failed(maybeCallerType))
           return failure();
-        callerType = *maybeCallerType;
+        assert(isa<BaseMemRefType>(*maybeCallerType) && "expected memref type");
+        callerType = cast<BaseMemRefType>(*maybeCallerType);
       }
 
       if (!bufferType) {
diff --git a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
index a57d58ab28d2..85d1b5ac73bf 100644
--- a/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -164,8 +164,8 @@ struct SelectOpInterface
     // buffers have different types, they differ only in their layout map. Cast
     // both of them to the most dynamic MemRef type.
     if (trueBuffer.getType() != falseBuffer.getType()) {
-      auto targetType =
-          bufferization::getBufferType(selectOp.getResult(), options, state);
+      auto targetType = bufferization::detail::asMemRefType(
+          bufferization::getBufferType(selectOp.getResult(), options, state));
       if (failed(targetType))
         return failure();
       if (trueBuffer.getType() != *targetType)
@@ -187,10 +187,12 @@ struct SelectOpInterface
                 SmallVector<Value> &invocationStack) const {
     auto selectOp = cast<arith::SelectOp>(op);
     assert(value == selectOp.getResult() && "invalid value");
-    auto trueType = bufferization::getBufferType(
-        selectOp.getTrueValue(), options, state, invocationStack);
-    auto falseType = bufferization::getBufferType(
-        selectOp.getFalseValue(), options, state, invocationStack);
+    auto trueType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            selectOp.getTrueValue(), options, state, invocationStack));
+    auto falseType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            selectOp.getFalseValue(), options, state, invocationStack));
     if (failed(trueType) || failed(falseType))
       return failure();
     if (*trueType == *falseType)
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
index dd43647682ea..2ab182c9b7b2 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp
@@ -171,7 +171,9 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   if (llvm::isa<RankedTensorType>(shapedValue.getType())) {
     tensor = shapedValue;
   } else if (llvm::isa<MemRefType>(shapedValue.getType())) {
-    tensor = b.create<ToTensorOp>(loc, shapedValue);
+    tensor = b.create<ToTensorOp>(
+        loc, memref::getTensorTypeFromMemRefType(shapedValue.getType()),
+        shapedValue);
   } else if (llvm::isa<UnrankedTensorType>(shapedValue.getType()) ||
              llvm::isa<UnrankedMemRefType>(shapedValue.getType())) {
     return getOwnerOfValue(shapedValue)
@@ -211,8 +213,8 @@ FailureOr<Value> bufferization::allocateTensorForShapedValue(
   // Add 'memory_space' attribute. Not needed if 'copy' operand is specified.
   if (copy)
     return allocTensorOp.getResult();
-  FailureOr<BaseMemRefType> copyBufferType =
-      getBufferType(tensor, options, state);
+  auto copyBufferType =
+      detail::asMemRefType(getBufferType(tensor, options, state));
   if (failed(copyBufferType))
     return failure();
   std::optional<Attribute> memorySpace = copyBufferType->getMemorySpace();
@@ -672,28 +674,28 @@ FailureOr<Value> bufferization::getBuffer(RewriterBase &rewriter, Value value,
                                           const BufferizationOptions &options,
                                           const BufferizationState &state) {
 #ifndef NDEBUG
-  auto tensorType = llvm::dyn_cast<TensorType>(value.getType());
+  auto tensorType = llvm::dyn_cast<TensorLikeType>(value.getType());
   assert(tensorType && "unexpected non-tensor type");
 #endif // NDEBUG
 
   // Replace "%t = to_tensor %m" with %m.
   if (auto toTensorOp = value.getDefiningOp<bufferization::ToTensorOp>())
-    return toTensorOp.getMemref();
+    return toTensorOp.getBuffer();
 
   // Insert to_buffer op.
   OpBuilder::InsertionGuard g(rewriter);
   setInsertionPointAfter(rewriter, value);
-  FailureOr<BaseMemRefType> memrefType = getBufferType(value, options, state);
-  if (failed(memrefType))
+  FailureOr<BufferLikeType> bufferType = getBufferType(value, options, state);
+  if (failed(bufferType))
     return failure();
-  ensureToBufferOpIsValid(value, *memrefType);
+  ensureToBufferOpIsValid(value, *bufferType);
   return rewriter
-      .create<bufferization::ToBufferOp>(value.getLoc(), *memrefType, value)
+      .create<bufferization::ToBufferOp>(value.getLoc(), *bufferType, value)
       .getResult();
 }
 
 /// Return the buffer type for a given Value (tensor) after bufferization.
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 bufferization::getBufferType(Value value, const BufferizationOptions &options,
                              const BufferizationState &state) {
   SmallVector<Value> invocationStack;
@@ -701,11 +703,11 @@ bufferization::getBufferType(Value value, const BufferizationOptions &options,
 }
 
 /// Return the buffer type for a given Value (tensor) after bufferization.
-FailureOr<BaseMemRefType>
+FailureOr<BufferLikeType>
 bufferization::getBufferType(Value value, const BufferizationOptions &options,
                              const BufferizationState &state,
                              SmallVector<Value> &invocationStack) {
-  assert(llvm::isa<TensorType>(value.getType()) &&
+  assert(llvm::isa<TensorLikeType>(value.getType()) &&
          "unexpected non-tensor type");
   invocationStack.push_back(value);
   auto popFromStack =
@@ -718,13 +720,9 @@ bufferization::getBufferType(Value value, const BufferizationOptions &options,
     return bufferizableOp.getBufferType(value, options, state, invocationStack);
 
   // Op is not bufferizable.
-  auto memSpace =
-      options.defaultMemorySpaceFn(cast<TensorType>(value.getType()));
-  if (!memSpace.has_value())
-    return op->emitError("could not infer memory space");
-
-  return getMemRefType(cast<TensorType>(value.getType()), options,
-                       /*layout=*/{}, *memSpace);
+  return cast<TensorLikeType>(value.getType()).getBufferType(options, [&]() {
+    return op->emitError();
+  });
 }
 
 bool bufferization::hasTensorSemantics(Operation *op) {
@@ -744,12 +742,11 @@ void bufferization::replaceOpWithBufferizedValues(RewriterBase &rewriter,
   SmallVector<Value> replacements;
   for (OpResult opResult : op->getOpResults()) {
     Value replacement = values[opResult.getResultNumber()];
-    if (llvm::isa<TensorType>(opResult.getType())) {
+    if (llvm::isa<TensorLikeType>(opResult.getType())) {
       // The OpResult is a tensor. Such values are replaced with memrefs during
       // bufferization.
-      assert((llvm::isa<MemRefType>(replacement.getType()) ||
-              llvm::isa<UnrankedMemRefType>(replacement.getType())) &&
-             "tensor op result should be replaced with a memref value");
+      assert(llvm::isa<BufferLikeType>(replacement.getType()) &&
+             "tensor op result should be replaced with a buffer value");
       // The existing uses of the OpResult still expect a tensor. Insert a
       // ToTensorOp. Throughout bufferization, this ToTensorOp will gradually
       // loose all of its users and eventually DCE away.
@@ -969,8 +966,8 @@ FailureOr<BaseMemRefType> bufferization::detail::defaultGetBufferType(
     // If the OpResult has an equivalent OpOperand, both OpResult and
     // OpOperand bufferize to the exact same buffer type.
     Value equivalentOperand = aliases.getAliases().front().opOperand->get();
-    return getBufferType(equivalentOperand, options, bufferizationState,
-                         invocationStack);
+    return asMemRefType(getBufferType(equivalentOperand, options,
+                                      bufferizationState, invocationStack));
   }
 
   // If we do not know the memory space and there is no default memory space,
@@ -1030,7 +1027,7 @@ bufferization::detail::unknownGetAliasingValues(OpOperand &opOperand) {
 }
 
 bool bufferization::detail::defaultHasTensorSemantics(Operation *op) {
-  auto isaTensor = [](Type t) { return isa<TensorType>(t); };
+  auto isaTensor = [](Type t) { return isa<TensorLikeType>(t); };
   bool hasTensorBlockArgument = any_of(op->getRegions(), [&](Region &r) {
     return any_of(r.getBlocks(), [&](Block &b) {
       return any_of(b.getArguments(), [&](BlockArgument bbArg) {
@@ -1045,3 +1042,19 @@ bool bufferization::detail::defaultHasTensorSemantics(Operation *op) {
     return true;
   return any_of(op->getOperandTypes(), isaTensor);
 }
+
+FailureOr<BaseMemRefType>
+bufferization::detail::asMemRefType(FailureOr<BufferLikeType> bufferType) {
+  if (failed(bufferType))
+    return failure();
+  return cast<BaseMemRefType>(*bufferType);
+}
+
+bool bufferization::detail::typesMatchAfterBufferization(Operation &op,
+                                                         Value tensor,
+                                                         Value buffer) {
+  return mlir::succeeded(
+      cast<TensorLikeType>(tensor.getType())
+          .verifyCompatibleBufferType(cast<BufferLikeType>(buffer.getType()),
+                                      [&]() { return op.emitError(); }));
+}
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
index d8eac01c2dea..6c08cdfb669f 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationDialect.cpp
@@ -57,7 +57,37 @@ struct BufferizationInlinerInterface : public DialectInlinerInterface {
 template <typename Tensor>
 struct BuiltinTensorExternalModel
     : TensorLikeType::ExternalModel<BuiltinTensorExternalModel<Tensor>,
-                                    Tensor> {};
+                                    Tensor> {
+  llvm::FailureOr<BufferLikeType> getBufferType(
+      mlir::Type tensor, const BufferizationOptions &options,
+      llvm::function_ref<mlir::InFlightDiagnostic()> emitError) const {
+    auto tensorType = cast<TensorType>(tensor);
+    auto memSpace = options.defaultMemorySpaceFn(tensorType);
+    if (!memSpace.has_value())
+      return emitError() << "could not infer memory space";
+
+    return cast<BufferLikeType>(
+        getMemRefType(tensorType, options, /*layout=*/{}, *memSpace));
+  }
+
+  mlir::LogicalResult verifyCompatibleBufferType(
+      mlir::Type tensor, BufferLikeType bufferType,
+      llvm::function_ref<mlir::InFlightDiagnostic()> emitError) const {
+    assert(isa<TensorType>(tensor) && "expected tensor type");
+    assert(isa<BaseMemRefType>(bufferType) && "expected memref type");
+
+    auto tensorType = cast<ShapedType>(tensor);
+    auto memrefType = cast<ShapedType>(bufferType);
+
+    if (tensorType.getShape() != memrefType.getShape())
+      return emitError() << "shapes do not match";
+
+    if (tensorType.getElementType() != memrefType.getElementType())
+      return emitError() << "element types do not match";
+
+    return mlir::success();
+  }
+};
 
 template <typename MemRef>
 struct BuiltinMemRefExternalModel
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
index dc54ac94aed3..9bd87d66c7d3 100644
--- a/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationOps.cpp
@@ -90,12 +90,12 @@ LogicalResult mlir::bufferization::foldToBufferToTensorPair(
   if (!bufferToTensor)
     return failure();
 
-  Type srcType = bufferToTensor.getMemref().getType();
+  Type srcType = bufferToTensor.getBuffer().getType();
   Type destType = toBuffer.getType();
 
   // Directly rewrite if the type did not change.
   if (srcType == destType) {
-    rewriter.replaceOp(toBuffer, bufferToTensor.getMemref());
+    rewriter.replaceOp(toBuffer, bufferToTensor.getBuffer());
     return success();
   }
 
@@ -106,7 +106,7 @@ LogicalResult mlir::bufferization::foldToBufferToTensorPair(
   // Ranked memref -> Ranked memref cast.
   if (rankedSrcType && rankedDestType) {
     FailureOr<Value> replacement = castOrReallocMemRefValue(
-        rewriter, bufferToTensor.getMemref(), rankedDestType, options);
+        rewriter, bufferToTensor.getBuffer(), rankedDestType, options);
     if (failed(replacement))
       return failure();
 
@@ -124,7 +124,7 @@ LogicalResult mlir::bufferization::foldToBufferToTensorPair(
   assert(memref::CastOp::areCastCompatible(srcType, destType) &&
          "expected that types are cast compatible");
   rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, destType,
-                                              bufferToTensor.getMemref());
+                                              bufferToTensor.getBuffer());
   return success();
 }
 
@@ -233,8 +233,9 @@ AllocTensorOp::getBufferType(Value value, const BufferizationOptions &options,
   if (getMemorySpace().has_value()) {
     memorySpace = *getMemorySpace();
   } else if (getCopy()) {
-    auto copyBufferType = bufferization::getBufferType(getCopy(), options,
-                                                       state, invocationStack);
+    auto copyBufferType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            getCopy(), options, state, invocationStack));
     if (failed(copyBufferType))
       return failure();
     memorySpace = copyBufferType->getMemorySpace();
@@ -642,8 +643,9 @@ Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
   assert(getRestrict() &&
          "expected that ops with memrefs dest have 'restrict'");
   setRestrict(false);
-  return builder.create<ToTensorOp>(loc, getDest(), /*restrict=*/true,
-                                    getWritable());
+  return builder.create<ToTensorOp>(
+      loc, memref::getTensorTypeFromMemRefType(getDest().getType()), getDest(),
+      /*restrict=*/true, getWritable());
 }
 
 bool MaterializeInDestinationOp::isEquivalentSubset(
@@ -744,7 +746,7 @@ bool ToTensorOp::isWritable(Value value, const AnalysisState &state) {
 }
 
 OpFoldResult ToTensorOp::fold(FoldAdaptor) {
-  if (auto toBuffer = getMemref().getDefiningOp<ToBufferOp>())
+  if (auto toBuffer = getBuffer().getDefiningOp<ToBufferOp>())
     // Approximate alias analysis by conservatively folding only when no there
     // is no interleaved operation.
     if (toBuffer->getBlock() == this->getOperation()->getBlock() &&
@@ -764,7 +766,7 @@ struct DimOfToTensorFolder : public OpRewritePattern<tensor::DimOp> {
       return failure();
 
     rewriter.replaceOpWithNewOp<memref::DimOp>(
-        dimOp, memrefToTensorOp.getMemref(), dimOp.getIndex());
+        dimOp, memrefToTensorOp.getBuffer(), dimOp.getIndex());
     return success();
   }
 };
@@ -781,8 +783,8 @@ void ToTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
 
 OpFoldResult ToBufferOp::fold(FoldAdaptor) {
   if (auto memrefToTensor = getTensor().getDefiningOp<ToTensorOp>())
-    if (memrefToTensor.getMemref().getType() == getType())
-      return memrefToTensor.getMemref();
+    if (memrefToTensor.getBuffer().getType() == getType())
+      return memrefToTensor.getBuffer();
   return {};
 }
 
diff --git a/mlir/lib/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp b/mlir/lib/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp
new file mode 100644
index 000000000000..0e973915c6fc
--- /dev/null
+++ b/mlir/lib/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp
@@ -0,0 +1,21 @@
+//===- BufferizationTypeInterfaces.cpp - Type Interfaces --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.h"
+
+//===----------------------------------------------------------------------===//
+// Bufferization Type Interfaces
+//===----------------------------------------------------------------------===//
+
+namespace mlir {
+namespace bufferization {
+
+#include "mlir/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp.inc"
+
+} // namespace bufferization
+} // namespace mlir
diff --git a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
index 63dcc1eb233e..5d8f0060f2c3 100644
--- a/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Bufferization/IR/CMakeLists.txt
@@ -6,6 +6,7 @@ add_mlir_dialect_library(MLIRBufferizationDialect
   BufferizationDialect.cpp
   BufferViewFlowOpInterface.cpp
   UnstructuredControlFlow.cpp
+  BufferizationTypeInterfaces.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/Bufferization
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
index 7e9b9119ce94..6472ef3eff2a 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/Bufferize.cpp
@@ -412,11 +412,11 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
       continue;
     }
 
-    FailureOr<BaseMemRefType> memrefType =
+    FailureOr<BufferLikeType> bufferType =
         bufferization::getBufferType(bbArg, options, state);
-    if (failed(memrefType))
+    if (failed(bufferType))
       return failure();
-    newTypes.push_back(*memrefType);
+    newTypes.push_back(*bufferType);
   }
 
   // Change the type of all block arguments.
@@ -463,7 +463,7 @@ bufferization::bufferizeBlockSignature(Block *block, RewriterBase &rewriter,
         newOperands.push_back(operand);
         continue;
       }
-      FailureOr<BaseMemRefType> operandBufferType =
+      FailureOr<BufferLikeType> operandBufferType =
           bufferization::getBufferType(operand, options, state);
       if (failed(operandBufferType))
         return failure();
diff --git a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
index a0168da44b7b..453ed43bcadd 100644
--- a/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Bufferization/Transforms/FuncBufferizableOpInterfaceImpl.cpp
@@ -255,7 +255,7 @@ struct CallOpInterface
       }
 
       // Returning a memref.
-      FailureOr<BaseMemRefType> resultType =
+      FailureOr<BufferLikeType> resultType =
           bufferization::getBufferType(result, options, state);
       if (failed(resultType))
         return failure();
@@ -290,13 +290,13 @@ struct CallOpInterface
         // The called function was not bufferized yet. This can happen when
         // there cycles in the function call graph. Compute the bufferized
         // result type.
-        FailureOr<BaseMemRefType> maybeMemRefType =
+        FailureOr<BufferLikeType> maybeBufferType =
             bufferization::getBufferType(
                 funcOp.getArgument(opOperand.getOperandNumber()), options,
                 state);
-        if (failed(maybeMemRefType))
+        if (failed(maybeBufferType))
           return failure();
-        memRefType = *maybeMemRefType;
+        memRefType = *maybeBufferType;
       }
 
       // Since we don't yet have a clear layout story, to_buffer may
diff --git a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
index 94a4b9011c16..573420f6a9aa 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/ConvertToDestinationStyle.cpp
@@ -252,7 +252,8 @@ Value linalg::bufferizeToAllocation(
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
   Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
-      loc, alloc, /*restrict=*/true, /*writable=*/true);
+      loc, padOp.getResult().getType(), alloc, /*restrict=*/true,
+      /*writable=*/true);
   rewriter.replaceOp(padOp, toTensorOp);
   return alloc;
 }
@@ -340,7 +341,8 @@ Value linalg::bufferizeToAllocation(
   // Create bufferization.to_tensor with "restrict" and "writable". The returned
   // tensor is a new buffer allocation, so it does not alias with any buffer.
   Value toTensorOp = rewriter.create<bufferization::ToTensorOp>(
-      loc, alloc, /*restrict=*/true, /*writable=*/true);
+      loc, allocTensorOp.getResult().getType(), alloc, /*restrict=*/true,
+      /*writable=*/true);
   rewriter.replaceOp(allocTensorOp, toTensorOp);
   return alloc;
 }
@@ -567,7 +569,8 @@ Value linalg::bufferizeToAllocation(
       createMemcpy(rewriter, op->getLoc(), operand->get(), alloc, options);
     }
     rewriter.modifyOpInPlace(op, [&]() {
-      auto toTensorOp = rewriter.create<ToTensorOp>(op->getLoc(), alloc);
+      auto toTensorOp = rewriter.create<ToTensorOp>(
+          op->getLoc(), operand->get().getType(), alloc);
       operand->set(toTensorOp);
       if (options.bufferizeDestinationOnly) {
         rewriter.modifyOpInPlace(toTensorOp, [&]() {
diff --git a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
index 46fa77a7dc4e..58562536be61 100644
--- a/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -108,7 +108,7 @@ struct ConditionOpInterface
             getBuffer(rewriter, value, options, state);
         if (failed(maybeBuffer))
           return failure();
-        FailureOr<BaseMemRefType> resultType = bufferization::getBufferType(
+        FailureOr<BufferLikeType> resultType = bufferization::getBufferType(
             whileOp.getAfterArguments()[it.index()], options, state);
         if (failed(resultType))
           return failure();
@@ -292,8 +292,9 @@ struct IfOpInterface
       // True branch was already bufferized.
       thenBufferType = cast<BaseMemRefType>(thenValue.getType());
     } else {
-      auto maybeBufferType = bufferization::getBufferType(
-          thenValue, options, state, invocationStack);
+      auto maybeBufferType =
+          bufferization::detail::asMemRefType(bufferization::getBufferType(
+              thenValue, options, state, invocationStack));
       if (failed(maybeBufferType))
         return failure();
       thenBufferType = *maybeBufferType;
@@ -302,8 +303,9 @@ struct IfOpInterface
       // False branch was already bufferized.
       elseBufferType = cast<BaseMemRefType>(elseValue.getType());
     } else {
-      auto maybeBufferType = bufferization::getBufferType(
-          elseValue, options, state, invocationStack);
+      auto maybeBufferType =
+          bufferization::detail::asMemRefType(bufferization::getBufferType(
+              elseValue, options, state, invocationStack));
       if (failed(maybeBufferType))
         return failure();
       elseBufferType = *maybeBufferType;
@@ -406,9 +408,7 @@ struct IndexSwitchOpInterface
         return bufferType;
       auto maybeBufferType = bufferization::getBufferType(
           yieldedValue, options, state, invocationStack);
-      if (failed(maybeBufferType))
-        return failure();
-      return maybeBufferType;
+      return bufferization::detail::asMemRefType(maybeBufferType);
     };
 
     // Compute buffer type of the default case.
@@ -527,8 +527,8 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
     const BufferizationOptions &options, const BufferizationState &state,
     SmallVector<Value> &invocationStack) {
   // Determine the buffer type of the init_arg.
-  auto initArgBufferType =
-      bufferization::getBufferType(initArg, options, state, invocationStack);
+  auto initArgBufferType = bufferization::detail::asMemRefType(
+      bufferization::getBufferType(initArg, options, state, invocationStack));
   if (failed(initArgBufferType))
     return failure();
 
@@ -554,8 +554,9 @@ static FailureOr<BaseMemRefType> computeLoopRegionIterArgBufferType(
   } else {
     // Note: This typically triggers a recursive call for the buffer type of
     // the iter_arg.
-    auto maybeBufferType = bufferization::getBufferType(yieldedValue, options,
-                                                        state, invocationStack);
+    auto maybeBufferType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            yieldedValue, options, state, invocationStack));
     if (failed(maybeBufferType))
       return failure();
     yieldedValueBufferType = *maybeBufferType;
@@ -718,8 +719,12 @@ struct ForOpInterface
     if (auto opResult = dyn_cast<OpResult>(value)) {
       // The type of an OpResult must match the corresponding iter_arg type.
       BlockArgument bbArg = forOp.getTiedLoopRegionIterArg(opResult);
-      return bufferization::getBufferType(bbArg, options, state,
-                                          invocationStack);
+      auto bufferType =
+          bufferization::getBufferType(bbArg, options, state, invocationStack);
+      if (failed(bufferType))
+        return failure();
+      assert(isa<BaseMemRefType>(*bufferType) && "expected memref type");
+      return cast<BaseMemRefType>(*bufferType);
     }
 
     // Compute result/argument number.
@@ -1078,8 +1083,8 @@ struct WhileOpInterface
       // scf.condition was already bufferized.
       return cast<BaseMemRefType>(conditionYieldedVal.getType());
     }
-    return bufferization::getBufferType(conditionYieldedVal, options, state,
-                                        invocationStack);
+    return bufferization::detail::asMemRefType(bufferization::getBufferType(
+        conditionYieldedVal, options, state, invocationStack));
   }
 
   /// Assert that yielded values of an scf.while op are equivalent to their
@@ -1185,14 +1190,14 @@ struct YieldOpInterface
         // We may have to cast the value before yielding it.
         if (isa<scf::ForOp, scf::IfOp, scf::IndexSwitchOp>(
                 yieldOp->getParentOp())) {
-          FailureOr<BaseMemRefType> resultType = bufferization::getBufferType(
+          FailureOr<BufferLikeType> resultType = bufferization::getBufferType(
               yieldOp->getParentOp()->getResult(it.index()), options, state);
           if (failed(resultType))
             return failure();
           buffer = castBuffer(rewriter, buffer, *resultType);
         } else if (auto whileOp =
                        dyn_cast<scf::WhileOp>(yieldOp->getParentOp())) {
-          FailureOr<BaseMemRefType> resultType = bufferization::getBufferType(
+          FailureOr<BufferLikeType> resultType = bufferization::getBufferType(
               whileOp.getBeforeArguments()[it.index()], options, state);
           if (failed(resultType))
             return failure();
@@ -1307,15 +1312,15 @@ struct ForallOpInterface
     if (auto bbArg = dyn_cast<BlockArgument>(value))
       // A tensor block argument has the same bufferized type as the
       // corresponding output operand.
-      return bufferization::getBufferType(
-          forallOp.getTiedOpOperand(bbArg)->get(), options, state,
-          invocationStack);
+      return bufferization::detail::asMemRefType(
+          bufferization::getBufferType(forallOp.getTiedOpOperand(bbArg)->get(),
+                                       options, state, invocationStack));
 
     // The bufferized result type is the same as the bufferized type of the
     // corresponding output operand.
-    return bufferization::getBufferType(
+    return bufferization::detail::asMemRefType(bufferization::getBufferType(
         forallOp.getOutputs()[cast<OpResult>(value).getResultNumber()], options,
-        state, invocationStack);
+        state, invocationStack));
   }
 
   bool isRepetitiveRegion(Operation *op, unsigned index) const {
diff --git a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
index dc91117a5193..8a471c12d21e 100644
--- a/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Shape/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -67,7 +67,7 @@ struct AssumingOpInterface
     for (const auto &it : llvm::enumerate(assumingOp->getResultTypes())) {
       if (isa<TensorType>(it.value())) {
         newResults.push_back(rewriter.create<bufferization::ToTensorOp>(
-            assumingOp.getLoc(), newOp->getResult(it.index())));
+            assumingOp.getLoc(), it.value(), newOp->getResult(it.index())));
       } else {
         newResults.push_back(newOp->getResult(it.index()));
       }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index e5f2418367a5..e89b34d457ff 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -651,7 +651,7 @@ static LogicalResult rewriteSpMV(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, memY);
+  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, y.getType(), memY);
   return success();
 }
 
@@ -752,7 +752,7 @@ static LogicalResult rewriteSpMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, bufC);
+  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, c.getType(), bufC);
   return success();
 }
 
@@ -925,9 +925,12 @@ static LogicalResult rewriteSpGEMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  Value vt = rewriter.create<bufferization::ToTensorOp>(loc, valH);
-  Value rt = rewriter.create<bufferization::ToTensorOp>(loc, rowH);
-  Value ct = rewriter.create<bufferization::ToTensorOp>(loc, colH);
+  Value vt = rewriter.create<bufferization::ToTensorOp>(
+      loc, memref::getTensorTypeFromMemRefType(valH.getType()), valH);
+  Value rt = rewriter.create<bufferization::ToTensorOp>(
+      loc, memref::getTensorTypeFromMemRefType(rowH.getType()), rowH);
+  Value ct = rewriter.create<bufferization::ToTensorOp>(
+      loc, memref::getTensorTypeFromMemRefType(colH.getType()), colH);
   rewriter.replaceOpWithNewOp<AssembleOp>(op, c.getType(), ValueRange{rt, ct},
                                           vt);
   return success();
@@ -1043,7 +1046,7 @@ static LogicalResult rewrite2To4SpMM(PatternRewriter &rewriter,
   tokens.clear();
 
   // Done.
-  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, bufC);
+  rewriter.replaceOpWithNewOp<bufferization::ToTensorOp>(op, C.getType(), bufC);
   return success();
 }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index e5f9717c3fba..14ced56b8365 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -1471,7 +1471,8 @@ struct SparseDisassembleOpConverter
     // Converts MemRefs back to Tensors.
     SmallVector<Value> retValues = llvm::to_vector(
         llvm::map_range(retMem, [&rewriter, loc](Value v) -> Value {
-          return rewriter.create<bufferization::ToTensorOp>(loc, v);
+          return rewriter.create<bufferization::ToTensorOp>(
+              loc, memref::getTensorTypeFromMemRefType(v.getType()), v);
         }));
     // Appends the actual memory length used in each buffer returned.
     retValues.append(retLen.begin(), retLen.end());
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index 9ffa64dc821d..7f0b65768744 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -867,7 +867,9 @@ public:
     // Converts MemRefs back to Tensors.
     assert(retVal.size() + retLen.size() == op.getNumResults());
     for (unsigned i = 0, sz = retVal.size(); i < sz; i++) {
-      auto tensor = rewriter.create<bufferization::ToTensorOp>(loc, retVal[i]);
+      auto tensor = rewriter.create<bufferization::ToTensorOp>(
+          loc, memref::getTensorTypeFromMemRefType(retVal[i].getType()),
+          retVal[i]);
       retVal[i] =
           rewriter.create<tensor::CastOp>(loc, op.getResultTypes()[i], tensor);
     }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
index 57291064eba2..1bd9563b3db0 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
@@ -549,8 +549,8 @@ TypedValue<BaseMemRefType>
 sparse_tensor::genToMemref(OpBuilder &builder, Location loc, Value tensor) {
   auto tTp = llvm::cast<TensorType>(tensor.getType());
   auto mTp = MemRefType::get(tTp.getShape(), tTp.getElementType());
-  return builder.create<bufferization::ToBufferOp>(loc, mTp, tensor)
-      .getResult();
+  return cast<TypedValue<BaseMemRefType>>(
+      builder.create<bufferization::ToBufferOp>(loc, mTp, tensor).getResult());
 }
 
 Value sparse_tensor::createOrFoldSliceOffsetOp(OpBuilder &builder, Location loc,
diff --git a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
index 4b778b768d13..729c048db456 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/BufferizableOpInterfaceImpl.cpp
@@ -54,8 +54,9 @@ struct CastOpInterface
                 const BufferizationState &state,
                 SmallVector<Value> &invocationStack) const {
     auto castOp = cast<tensor::CastOp>(op);
-    auto maybeSrcBufferType = bufferization::getBufferType(
-        castOp.getSource(), options, state, invocationStack);
+    auto maybeSrcBufferType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            castOp.getSource(), options, state, invocationStack));
     if (failed(maybeSrcBufferType))
       return failure();
     Attribute memorySpace = maybeSrcBufferType->getMemorySpace();
@@ -500,8 +501,8 @@ struct FromElementsOpInterface
         /*copy=*/false);
     if (failed(tensorAlloc))
       return failure();
-    FailureOr<BaseMemRefType> memrefType =
-        bufferization::getBufferType(*tensorAlloc, options, state);
+    FailureOr<BaseMemRefType> memrefType = bufferization::detail::asMemRefType(
+        bufferization::getBufferType(*tensorAlloc, options, state));
     if (failed(memrefType))
       return failure();
     Value buffer = rewriter.create<bufferization::ToBufferOp>(
@@ -758,8 +759,9 @@ struct PadOpInterface
                 SmallVector<Value> &invocationStack) const {
     // Infer memory space from the source tensor.
     auto padOp = cast<tensor::PadOp>(op);
-    auto maybeSrcBufferType = bufferization::getBufferType(
-        padOp.getSource(), options, state, invocationStack);
+    auto maybeSrcBufferType =
+        bufferization::detail::asMemRefType(bufferization::getBufferType(
+            padOp.getSource(), options, state, invocationStack));
     if (failed(maybeSrcBufferType))
       return failure();
     MemRefLayoutAttrInterface layout;
diff --git a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
index cd19e3a5e82a..da3c26ce36ba 100644
--- a/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
+++ b/mlir/test/Dialect/Bufferization/Transforms/one-shot-bufferize.mlir
@@ -268,4 +268,23 @@ func.func @materialize_in_dest_raw(%f: f32, %f2: f32, %idx: index) -> (tensor<5x
   %r = tensor.extract %dest_filled[%idx] : tensor<5xf32>
 
   return %0, %r : tensor<5xf32>, f32
-}
\ No newline at end of file
+}
+
+// -----
+
+// CHECK-LABEL: func.func @test_dialect_op(
+// CHECK-SAME:    %[[ARG:.*]]: !test.test_tensor<[32, 64], f64>
+// CHECK-SAME:  ) -> !test.test_tensor<[32, 128], f64> {
+func.func @test_dialect_op(%arg: !test.test_tensor<[32, 64], f64>)
+    -> !test.test_tensor<[32, 128], f64> {
+  // CHECK: %[[MEMREF:.*]] = bufferization.to_buffer %[[ARG]]
+  // CHECK: %[[DUMMY:.*]] = "test.dummy_memref_op"(%[[MEMREF]])
+  // CHECK-SAME: : (!test.test_memref<[32, 64], f64>)
+  // CHECK-SAME: -> !test.test_memref<[32, 128], f64>
+  // CHECK: %[[OUT:.*]] = bufferization.to_tensor %[[DUMMY]]
+  %out = "test.dummy_tensor_op"(%arg) : (!test.test_tensor<[32, 64], f64>)
+    -> !test.test_tensor<[32, 128], f64>
+
+  // CHECK: return %[[OUT]]
+  return %out : !test.test_tensor<[32, 128], f64>
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
index b5a8bd10d6b6..78e44c6ec7a9 100644
--- a/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
+++ b/mlir/test/lib/Dialect/Test/TestOpDefs.cpp
@@ -8,6 +8,7 @@
 
 #include "TestDialect.h"
 #include "TestOps.h"
+#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/IR/Verifier.h"
 #include "mlir/Interfaces/FunctionImplementation.h"
@@ -1387,3 +1388,25 @@ TestMultiSlotAlloca::handleDestructuringComplete(
     const DestructurableMemorySlot &slot, OpBuilder &builder) {
   return createNewMultiAllocaWithoutSlot(slot, builder, *this);
 }
+
+::mlir::LogicalResult test::TestDummyTensorOp::bufferize(
+    ::mlir::RewriterBase &rewriter,
+    const ::mlir::bufferization::BufferizationOptions &options,
+    ::mlir::bufferization::BufferizationState &state) {
+  auto buffer =
+      mlir::bufferization::getBuffer(rewriter, getInput(), options, state);
+  if (mlir::failed(buffer))
+    return failure();
+
+  const auto outType = getOutput().getType();
+  const auto bufferizedOutType = test::TestMemrefType::get(
+      getContext(), outType.getShape(), outType.getElementType(), nullptr);
+  // replace op with memref analogy
+  auto dummyMemrefOp = rewriter.create<test::TestDummyMemrefOp>(
+      getLoc(), bufferizedOutType, *buffer);
+
+  mlir::bufferization::replaceOpWithBufferizedValues(rewriter, getOperation(),
+                                                     dummyMemrefOp.getResult());
+
+  return mlir::success();
+}
diff --git a/mlir/test/lib/Dialect/Test/TestOps.h b/mlir/test/lib/Dialect/Test/TestOps.h
index c2ee5f9ab9a5..b414b47c8742 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.h
+++ b/mlir/test/lib/Dialect/Test/TestOps.h
@@ -13,6 +13,7 @@
 #include "TestInterfaces.h"
 #include "TestTypes.h"
 #include "mlir/Bytecode/BytecodeImplementation.h"
+#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/DLTI/DLTI.h"
 #include "mlir/Dialect/DLTI/Traits.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 59330fdb1bb2..79bcd9c2e0a9 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -31,7 +31,7 @@ include "mlir/Interfaces/InferTypeOpInterface.td"
 include "mlir/Interfaces/LoopLikeInterface.td"
 include "mlir/Interfaces/MemorySlotInterfaces.td"
 include "mlir/Interfaces/SideEffectInterfaces.td"
-
+include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.td"
 
 // Include the attribute definitions.
 include "TestAttrDefs.td"
@@ -2825,7 +2825,7 @@ def TestNVVMRequiresSMArchCondOp :
   let assemblyFormat = "attr-dict";
 }
 
-def TestNVVMRequirestSMArchCondMultiOp : 
+def TestNVVMRequirestSMArchCondMultiOp :
     TEST_Op<"nvvm_requires_sm_90a_or_sm_100a", [NVVMRequiresSMa<[90, 100]>]> {
   let arguments = (ins );
   let assemblyFormat = "attr-dict";
@@ -3552,4 +3552,58 @@ def TestAllocWithMultipleResults : TEST_Op<"alloc_with_multiple_results"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// Test Ops bufferization
+//===----------------------------------------------------------------------===//
+
+def TestDummyTensorOp : TEST_Op<"dummy_tensor_op", [BufferizableOpInterface]> {
+  let arguments = (ins
+    Arg<TestTensorType>:$input
+  );
+  let results = (outs
+    Arg<TestTensorType>:$output
+  );
+  let extraClassDeclaration = [{
+    // BufferizableOpInterface
+    bool bufferizesToMemoryRead(mlir::OpOperand&,
+      const mlir::bufferization::AnalysisState&);
+
+    bool bufferizesToMemoryWrite(mlir::OpOperand&,
+      const mlir::bufferization::AnalysisState&);
+
+    mlir::bufferization::AliasingValueList getAliasingValues(mlir::OpOperand&,
+      const mlir::bufferization::AnalysisState&);
+
+    mlir::LogicalResult bufferize(
+      mlir::RewriterBase& rewriter,
+      const mlir::bufferization::BufferizationOptions& options,
+      mlir::bufferization::BufferizationState &state);
+  }];
+
+  let extraClassDefinition = [{
+    bool test::TestDummyTensorOp::bufferizesToMemoryRead(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    bool test::TestDummyTensorOp::bufferizesToMemoryWrite(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return true;
+    }
+    ::mlir::bufferization::AliasingValueList
+    test::TestDummyTensorOp::getAliasingValues(::mlir::OpOperand&,
+        const ::mlir::bufferization::AnalysisState&) {
+      return {};
+    }
+  }];
+}
+
+def TestDummyMemrefOp : TEST_Op<"dummy_memref_op", []> {
+  let arguments = (ins
+    Arg<TestMemrefType>:$input
+  );
+  let results = (outs
+    Arg<TestMemrefType>:$output
+  );
+}
+
 #endif // TEST_OPS
diff --git a/mlir/test/lib/Dialect/Test/TestTypeDefs.td b/mlir/test/lib/Dialect/Test/TestTypeDefs.td
index 09294e84960f..03261f37c815 100644
--- a/mlir/test/lib/Dialect/Test/TestTypeDefs.td
+++ b/mlir/test/lib/Dialect/Test/TestTypeDefs.td
@@ -428,6 +428,15 @@ def TestTensorType : Test_Type<"TestTensor",
       return test::TestTensorType::get(
         getContext(), shape.value_or(getShape()), elementType);
     }
+
+    // TensorLikeTypeInterface:
+    ::mlir::FailureOr<::mlir::bufferization::BufferLikeType>
+    getBufferType(const ::mlir::bufferization::BufferizationOptions& options,
+                  ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError);
+
+    ::mlir::LogicalResult verifyCompatibleBufferType(
+        ::mlir::bufferization::BufferLikeType bufferType,
+        ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError);
   }];
 }
 
diff --git a/mlir/test/lib/Dialect/Test/TestTypes.cpp b/mlir/test/lib/Dialect/Test/TestTypes.cpp
index 5c784dcee6e1..2fc2f90ef6bc 100644
--- a/mlir/test/lib/Dialect/Test/TestTypes.cpp
+++ b/mlir/test/lib/Dialect/Test/TestTypes.cpp
@@ -545,3 +545,23 @@ TestTypeOpAsmTypeInterfaceType::getAlias(::llvm::raw_ostream &os) const {
   os << "op_asm_type_interface_type";
   return ::mlir::OpAsmDialectInterface::AliasResult::FinalAlias;
 }
+
+::mlir::FailureOr<::mlir::bufferization::BufferLikeType>
+TestTensorType::getBufferType(
+    const ::mlir::bufferization::BufferizationOptions &,
+    ::llvm::function_ref<::mlir::InFlightDiagnostic()>) {
+  return cast<bufferization::BufferLikeType>(
+      TestMemrefType::get(getContext(), getShape(), getElementType(), nullptr));
+}
+
+::mlir::LogicalResult TestTensorType::verifyCompatibleBufferType(
+    ::mlir::bufferization::BufferLikeType bufferType,
+    ::llvm::function_ref<::mlir::InFlightDiagnostic()> emitError) {
+  auto testMemref = dyn_cast<TestMemrefType>(bufferType);
+  if (!testMemref)
+    return emitError() << "expected TestMemrefType";
+
+  const bool valid = getShape() == testMemref.getShape() &&
+                     getElementType() == testMemref.getElementType();
+  return mlir::success(valid);
+}
-- 
cgit v1.2.3


From 6f4add34801e6ce02a5ebc96df4d1ca479125649 Mon Sep 17 00:00:00 2001
From: Omair Javaid <omair.javaid@linaro.org>
Date: Wed, 18 Jun 2025 19:23:54 +0500
Subject: [compiler-rt] [Fuzzer] Fix ARMv7 test link failure by linking
 unwinder (#144495)

compiler-rt/lib/fuzzer/tests build was failing on armv7, with undefined
references to unwinder symbols, such as __aeabi_unwind_cpp_pr0.

This occurs because the test is built with `-nostdlib++` but `libunwind`
is not explicitly linked to the final test executable.

This patch resolves the issue by adding CMake logic to explicitly link
the required unwinder to the fuzzer tests, inspired by the same solution
used to fix Scudo build failures by https://reviews.llvm.org/D142888.
---
 compiler-rt/lib/fuzzer/tests/CMakeLists.txt | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
index adfae3d63e64..543f486a9d50 100644
--- a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
+++ b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
@@ -35,6 +35,27 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
    COMPILER_RT_LIBCXXABI_PATH)
   list(APPEND LIBFUZZER_UNITTEST_CFLAGS -nostdinc++ -fno-exceptions)
   list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS -nostdlib++ -fno-exceptions)
+
+  # When we use -nostdlib++, we remove the default C++ runtime which normally
+  # provides the stack unwinding symbols (like __aeabi_unwind_cpp_pr0).
+  # We must now manually find and link a suitable unwinder library.
+  set(FUZZER_UNWINDER_LIBS)
+  if(COMPILER_RT_USE_LLVM_UNWINDER)
+    # Prefer LLVM's own libunwind.
+    list(APPEND FUZZER_UNWINDER_LIBS ${COMPILER_RT_UNWINDER_LINK_LIBS})
+  elseif(COMPILER_RT_HAS_GCC_S_LIB)
+    # As a fallback, use the shared libgcc_s library.
+    list(APPEND FUZZER_UNWINDER_LIBS gcc_s)
+  elseif(COMPILER_RT_HAS_GCC_LIB)
+    # As a final fallback, use the static libgcc library.
+    list(APPEND FUZZER_UNWINDER_LIBS gcc)
+  elseif(NOT COMPILER_RT_USE_BUILTINS_LIBRARY)
+    # If no unwinder is found and we aren't using the builtins library
+    message(FATAL_ERROR "Fuzzer tests require a suitable unwinder, but none was found.")
+  endif()
+  # Add the detected unwinder library to our link flags.
+  list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS ${FUZZER_UNWINDER_LIBS})
+
 endif()
 
 if ("-fvisibility=hidden" IN_LIST LIBFUZZER_CFLAGS)
-- 
cgit v1.2.3


From 36038a1048b2aab87ed18f982e960c044ad97670 Mon Sep 17 00:00:00 2001
From: Orlando Cazalet-Hyams <orlando.hyams@sony.com>
Date: Wed, 18 Jun 2025 16:04:18 +0100
Subject: [RemoveDIs][NFC] Remove dbg intrinsic handling code from SelectionDAG
 ISel (#144702)

---
 llvm/include/llvm/CodeGen/FunctionLoweringInfo.h   |  4 +-
 llvm/lib/CodeGen/SelectionDAG/FastISel.cpp         | 45 ----------------
 .../CodeGen/SelectionDAG/FunctionLoweringInfo.cpp  |  1 -
 .../CodeGen/SelectionDAG/SelectionDAGBuilder.cpp   | 63 ----------------------
 .../lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h |  1 -
 llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp |  4 --
 6 files changed, 1 insertion(+), 117 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
index 392da3f512df..b892a0e222a4 100644
--- a/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/llvm/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -35,7 +35,6 @@ namespace llvm {
 class Argument;
 class BasicBlock;
 class BranchProbabilityInfo;
-class DbgDeclareInst;
 class Function;
 class Instruction;
 class MachineFunction;
@@ -191,9 +190,8 @@ public:
   /// The current call site index being processed, if any. 0 if none.
   unsigned CurCallSite = 0;
 
-  /// Collection of dbg.declare instructions handled after argument
+  /// Collection of dbg_declare instructions handled after argument
   /// lowering and before ISel proper.
-  SmallPtrSet<const DbgDeclareInst *, 8> PreprocessedDbgDeclares;
   SmallPtrSet<const DbgVariableRecord *, 8> PreprocessedDVRDeclares;
 
   /// set - Initialize this FunctionLoweringInfo with the given Function
diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
index e8a3df3366b2..fb9eff942a46 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -1395,51 +1395,6 @@ bool FastISel::selectIntrinsicCall(const IntrinsicInst *II) {
   // Neither does the llvm.experimental.noalias.scope.decl intrinsic
   case Intrinsic::experimental_noalias_scope_decl:
     return true;
-  case Intrinsic::dbg_declare: {
-    const DbgDeclareInst *DI = cast<DbgDeclareInst>(II);
-    assert(DI->getVariable() && "Missing variable");
-    if (FuncInfo.PreprocessedDbgDeclares.contains(DI))
-      return true;
-
-    const Value *Address = DI->getAddress();
-    if (!lowerDbgDeclare(Address, DI->getExpression(), DI->getVariable(),
-                         MIMD.getDL()))
-      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI);
-
-    return true;
-  }
-  case Intrinsic::dbg_assign:
-    // A dbg.assign is a dbg.value with more information, typically produced
-    // during optimisation. If one reaches fastisel then something odd has
-    // happened (such as an optimised function being always-inlined into an
-    // optnone function). We will not be using the extra information in the
-    // dbg.assign in that case, just use its dbg.value fields.
-    [[fallthrough]];
-  case Intrinsic::dbg_value: {
-    // This form of DBG_VALUE is target-independent.
-    const DbgValueInst *DI = cast<DbgValueInst>(II);
-    const Value *V = DI->getValue();
-    DIExpression *Expr = DI->getExpression();
-    DILocalVariable *Var = DI->getVariable();
-    if (DI->hasArgList())
-      // Signal that we don't have a location for this.
-      V = nullptr;
-
-    assert(Var->isValidLocationForIntrinsic(MIMD.getDL()) &&
-           "Expected inlined-at fields to agree");
-
-    if (!lowerDbgValue(V, Expr, Var, MIMD.getDL()))
-      LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n");
-
-    return true;
-  }
-  case Intrinsic::dbg_label: {
-    const DbgLabelInst *DI = cast<DbgLabelInst>(II);
-    assert(DI->getLabel() && "Missing label");
-    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
-            TII.get(TargetOpcode::DBG_LABEL)).addMetadata(DI->getLabel());
-    return true;
-  }
   case Intrinsic::objectsize:
     llvm_unreachable("llvm.objectsize.* should have been lowered already");
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index d4ed158729ca..098005b6adfa 100644
--- a/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -369,7 +369,6 @@ void FunctionLoweringInfo::clear() {
   StatepointStackSlots.clear();
   StatepointRelocationMaps.clear();
   PreferredExtendType.clear();
-  PreprocessedDbgDeclares.clear();
   PreprocessedDVRDeclares.clear();
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ec0c5473b0db..c01f1e792847 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6674,69 +6674,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
     DAG.setRoot(Res.getValue(1));
     return;
   }
-  case Intrinsic::dbg_declare: {
-    const auto &DI = cast<DbgDeclareInst>(I);
-    // Debug intrinsics are handled separately in assignment tracking mode.
-    // Some intrinsics are handled right after Argument lowering.
-    if (AssignmentTrackingEnabled ||
-        FuncInfo.PreprocessedDbgDeclares.count(&DI))
-      return;
-    LLVM_DEBUG(dbgs() << "SelectionDAG visiting dbg_declare: " << DI << "\n");
-    DILocalVariable *Variable = DI.getVariable();
-    DIExpression *Expression = DI.getExpression();
-    dropDanglingDebugInfo(Variable, Expression);
-    // Assume dbg.declare can not currently use DIArgList, i.e.
-    // it is non-variadic.
-    assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList");
-    handleDebugDeclare(DI.getVariableLocationOp(0), Variable, Expression,
-                       DI.getDebugLoc());
-    return;
-  }
-  case Intrinsic::dbg_label: {
-    const DbgLabelInst &DI = cast<DbgLabelInst>(I);
-    DILabel *Label = DI.getLabel();
-    assert(Label && "Missing label");
-
-    SDDbgLabel *SDV;
-    SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
-    DAG.AddDbgLabel(SDV);
-    return;
-  }
-  case Intrinsic::dbg_assign: {
-    // Debug intrinsics are handled separately in assignment tracking mode.
-    if (AssignmentTrackingEnabled)
-      return;
-    // If assignment tracking hasn't been enabled then fall through and treat
-    // the dbg.assign as a dbg.value.
-    [[fallthrough]];
-  }
-  case Intrinsic::dbg_value: {
-    // Debug intrinsics are handled separately in assignment tracking mode.
-    if (AssignmentTrackingEnabled)
-      return;
-    const DbgValueInst &DI = cast<DbgValueInst>(I);
-    assert(DI.getVariable() && "Missing variable");
-
-    DILocalVariable *Variable = DI.getVariable();
-    DIExpression *Expression = DI.getExpression();
-    dropDanglingDebugInfo(Variable, Expression);
-
-    if (DI.isKillLocation()) {
-      handleKillDebugValue(Variable, Expression, DI.getDebugLoc(), SDNodeOrder);
-      return;
-    }
-
-    SmallVector<Value *, 4> Values(DI.getValues());
-    if (Values.empty())
-      return;
-
-    bool IsVariadic = DI.hasArgList();
-    if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(),
-                          SDNodeOrder, IsVariadic))
-      addDanglingDebugInfo(Values, Variable, Expression, IsVariadic,
-                           DI.getDebugLoc(), SDNodeOrder);
-    return;
-  }
 
   case Intrinsic::eh_typeid_for: {
     // Find the type id for the given typeinfo.
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 35c15bc269d4..1c278076a219 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -56,7 +56,6 @@ class CleanupPadInst;
 class CleanupReturnInst;
 class Constant;
 class ConstrainedFPIntrinsic;
-class DbgValueInst;
 class DataLayout;
 class DIExpression;
 class DILocalVariable;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index ac6d25f141ec..4b98d87fcc63 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -1592,10 +1592,6 @@ static bool processDbgDeclare(FunctionLoweringInfo &FuncInfo,
 /// in case the declarations refer to arguments.
 static void processDbgDeclares(FunctionLoweringInfo &FuncInfo) {
   for (const auto &I : instructions(*FuncInfo.Fn)) {
-    const auto *DI = dyn_cast<DbgDeclareInst>(&I);
-    if (DI && processDbgDeclare(FuncInfo, DI->getAddress(), DI->getExpression(),
-                                DI->getVariable(), DI->getDebugLoc()))
-      FuncInfo.PreprocessedDbgDeclares.insert(DI);
     for (const DbgVariableRecord &DVR : filterDbgVars(I.getDbgRecordRange())) {
       if (DVR.Type == DbgVariableRecord::LocationType::Declare &&
           processDbgDeclare(FuncInfo, DVR.getVariableLocationOp(0),
-- 
cgit v1.2.3


From 8fc20bffabe7fe6cdc4a9ec1bc79202eba5f1f23 Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Wed, 18 Jun 2025 16:07:56 +0100
Subject: Fix bazel build issue caused by 142986 (#144721)

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel      | 2 ++
 utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 0b4441c15794..48f2d0900d3e 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -12630,6 +12630,7 @@ td_library(
         ":AllocationOpInterfaceTdFiles",
         ":BufferViewFlowOpInterfaceTdFiles",
         ":BufferizableOpInterfaceTdFiles",
+        ":BufferizationTypeInterfacesTdFiles",
         ":CopyOpInterfaceTdFiles",
         ":DestinationStyleOpInterfaceTdFiles",
         ":InferTypeOpInterfaceTdFiles",
@@ -12811,6 +12812,7 @@ cc_library(
         ":BufferDeallocationOpInterfaceIncGen",
         ":BufferViewFlowOpInterfaceIncGen",
         ":BufferizableOpInterfaceIncGen",
+        ":BufferizationTypeInterfacesIncGen",
         ":BufferizationBaseIncGen",
         ":BufferizationInterfaces",
         ":BufferizationOpsIncGen",
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index a2fb5ade7324..0eaf86da7f27 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -111,6 +111,7 @@ td_library(
     name = "TestOpTdFiles",
     srcs = glob(["lib/Dialect/Test/*.td"]),
     deps = [
+        "//mlir:BufferizableOpInterfaceTdFiles",
         "//mlir:BufferizationTypeInterfacesTdFiles",
         "//mlir:BuiltinDialectTdFiles",
         "//mlir:CallInterfacesTdFiles",
@@ -242,6 +243,7 @@ gentbl_cc_library(
     test = True,
     deps = [
         ":TestOpTdFiles",
+        "//mlir:BufferizableOpInterfaceTdFiles",
         "//mlir:BufferizationTypeInterfacesTdFiles",
         "//mlir:BuiltinDialectTdFiles",
     ],
-- 
cgit v1.2.3


From e4c3b037bc7f5d9a8089de4c509d3e6034735891 Mon Sep 17 00:00:00 2001
From: amordo <iammorjj@gmail.com>
Date: Wed, 18 Jun 2025 17:12:31 +0200
Subject: [InstCombine] Fold `tan(x) * cos(x) => sin(x)` (#136319)

This patch enables folding `tan(x) * cos(x) -> sin(x)` under the `contract` flag.

Fixes https://github.com/llvm/llvm-project/issues/34950.
---
 .../InstCombine/InstCombineMulDivRem.cpp           |  12 ++
 llvm/test/Transforms/InstCombine/fmul-tan-cos.ll   | 182 +++++++++++++++++++++
 2 files changed, 194 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/fmul-tan-cos.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 457199a72510..fcf4613b5d13 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1072,6 +1072,18 @@ Instruction *InstCombinerImpl::visitFMul(BinaryOperator &I) {
     return Result;
   }
 
+  // tan(X) * cos(X) -> sin(X)
+  if (I.hasAllowContract() &&
+      match(&I,
+            m_c_FMul(m_OneUse(m_Intrinsic<Intrinsic::tan>(m_Value(X))),
+                     m_OneUse(m_Intrinsic<Intrinsic::cos>(m_Deferred(X)))))) {
+    auto *Sin = Builder.CreateUnaryIntrinsic(Intrinsic::sin, X, &I);
+    if (auto *Metadata = I.getMetadata(LLVMContext::MD_fpmath)) {
+      Sin->setMetadata(LLVMContext::MD_fpmath, Metadata);
+    }
+    return replaceInstUsesWith(I, Sin);
+  }
+
   return nullptr;
 }
 
diff --git a/llvm/test/Transforms/InstCombine/fmul-tan-cos.ll b/llvm/test/Transforms/InstCombine/fmul-tan-cos.ll
new file mode 100644
index 000000000000..a85661f14670
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/fmul-tan-cos.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=instcombine < %s | FileCheck %s
+
+define double @fmul_tan_cos(double %a) {
+; CHECK-LABEL: define double @fmul_tan_cos(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[TAN:%.*]] = call double @llvm.tan.f64(double [[A]])
+; CHECK-NEXT:    [[COS:%.*]] = call double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[RES:%.*]] = fmul double [[TAN]], [[COS]]
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call double @llvm.tan.f64(double %a)
+  %cos = call double @llvm.cos.f64(double %a)
+  %res = fmul double %tan, %cos
+  ret double %res
+}
+
+define double @fmul_strict_tan_strict_cos_contract(double %a) {
+; CHECK-LABEL: define double @fmul_strict_tan_strict_cos_contract(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[TAN:%.*]] = call double @llvm.tan.f64(double [[A]])
+; CHECK-NEXT:    [[COS:%.*]] = call contract double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[RES:%.*]] = fmul double [[TAN]], [[COS]]
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %res = fmul double %tan, %cos
+  ret double %res
+}
+
+define double @fmul_contract_tan_strict_cos_strict(double %a) {
+; CHECK-LABEL: define double @fmul_contract_tan_strict_cos_strict(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call double @llvm.tan.f64(double %a)
+  %cos = call double @llvm.cos.f64(double %a)
+  %res = fmul contract double %tan, %cos
+  ret double %res
+}
+
+define double @fmul_contract_tan_contract_cos_strict(double %a) {
+; CHECK-LABEL: define double @fmul_contract_tan_contract_cos_strict(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call double @llvm.cos.f64(double %a)
+  %res = fmul contract double %tan, %cos
+  ret double %res
+}
+
+define double @fmul_tan_cos_contract_multiple_uses(double %a) {
+; CHECK-LABEL: define double @fmul_tan_cos_contract_multiple_uses(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[TAN:%.*]] = call contract double @llvm.tan.f64(double [[A]])
+; CHECK-NEXT:    [[COS:%.*]] = call contract double @llvm.cos.f64(double [[A]])
+; CHECK-NEXT:    [[RES:%.*]] = fmul contract double [[TAN]], [[COS]]
+; CHECK-NEXT:    call void @use(double [[COS]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %res = fmul contract double %tan, %cos
+  call void @use(double %cos)
+  ret double %res
+}
+
+define double @fmul_tan_cos_contract(double %a) {
+; CHECK-LABEL: define double @fmul_tan_cos_contract(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %res = fmul contract double %tan, %cos
+  ret double %res
+}
+
+define float @fmul_tanf_cosf_contract(float %a) {
+; CHECK-LABEL: define float @fmul_tanf_cosf_contract(
+; CHECK-SAME: float [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract float @llvm.sin.f32(float [[A]])
+; CHECK-NEXT:    ret float [[RES]]
+;
+  %tan = call contract float @llvm.tan.f32(float %a)
+  %cos = call contract float @llvm.cos.f32(float %a)
+  %res = fmul contract float %tan, %cos
+  ret float %res
+}
+
+define fp128 @fmul_tanfp128_cosfp128_contract(fp128 %a) {
+; CHECK-LABEL: define fp128 @fmul_tanfp128_cosfp128_contract(
+; CHECK-SAME: fp128 [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract fp128 @llvm.sin.f128(fp128 [[A]])
+; CHECK-NEXT:    ret fp128 [[RES]]
+;
+  %tan = call contract fp128 @llvm.tan.fp128(fp128 %a)
+  %cos = call contract fp128 @llvm.cos.fp128(fp128 %a)
+  %res = fmul contract fp128 %tan, %cos
+  ret fp128 %res
+}
+
+
+define double @commutativity_cos_tan(double %a) {
+; CHECK-LABEL: define double @commutativity_cos_tan(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %res = fmul contract double %cos, %tan
+  ret double %res
+}
+
+
+define double @tan_cos_value_mismatch(double %a, double %b) {
+; CHECK-LABEL: define double @tan_cos_value_mismatch(
+; CHECK-SAME: double [[A:%.*]], double [[B:%.*]]) {
+; CHECK-NEXT:    [[TAN:%.*]] = call contract double @llvm.tan.f64(double [[A]])
+; CHECK-NEXT:    [[COS:%.*]] = call contract double @llvm.cos.f64(double [[B]])
+; CHECK-NEXT:    [[RES:%.*]] = fmul contract double [[TAN]], [[COS]]
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %b)
+  %res = fmul contract double %tan, %cos
+  ret double %res
+}
+
+
+define <2 x double> @fmul_tan_cos_vector(<2 x double> %a) {
+; CHECK-LABEL: define <2 x double> @fmul_tan_cos_vector(
+; CHECK-SAME: <2 x double> [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract <2 x double> @llvm.sin.v2f64(<2 x double> [[A]])
+; CHECK-NEXT:    ret <2 x double> [[RES]]
+;
+  %tan = call contract <2 x double> @llvm.tan.v2f64(<2 x double> %a)
+  %cos = call contract <2 x double> @llvm.cos.v2f64(<2 x double> %a)
+  %res = fmul contract <2 x double> %tan, %cos
+  ret <2 x double> %res
+}
+
+
+define double @fmul_tan_cos_nnan_preservation(double %a) {
+; CHECK-LABEL: define double @fmul_tan_cos_nnan_preservation(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call nnan contract double @llvm.sin.f64(double [[A]])
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %res = fmul contract nnan double %tan, %cos
+  ret double %res
+}
+
+
+define double @fmul_tan_cos_fpmath_metadata_preservation(double %a) {
+; CHECK-LABEL: define double @fmul_tan_cos_fpmath_metadata_preservation(
+; CHECK-SAME: double [[A:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call contract double @llvm.sin.f64(double [[A]]), !fpmath [[META0:![0-9]+]]
+; CHECK-NEXT:    ret double [[RES]]
+;
+  %tan = call contract double @llvm.tan.f64(double %a)
+  %cos = call contract double @llvm.cos.f64(double %a)
+  %res = fmul contract double %tan, %cos, !fpmath !0
+  ret double %res
+}
+
+declare void @use(double)
+
+!0 = !{ float 2.5 }
+
+
+;.
+; CHECK: [[META0]] = !{float 2.500000e+00}
+;.
-- 
cgit v1.2.3


From b53c1e4ee810ac21dab5d27413af1f31a6a4cbfa Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn@arm.com>
Date: Wed, 18 Jun 2025 16:16:52 +0100
Subject: [AArch64] Add ISel for postindex ld1/st1 in big-endian (#144387)

When big-endian we need to use ld1/st1 for vector loads and stores so
that we get the elements in the correct order, but this prevents
postindex addressing from being used. Fix this by adding the appropriate
ISel patterns, plus the relevant changes in ISelLowering and
ISelDAGToDAG to cause postindex addressing to be used.
---
 llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp |   52 +-
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp |   12 +
 llvm/lib/Target/AArch64/AArch64InstrInfo.td     |   84 +-
 llvm/test/CodeGen/AArch64/vector-ldst-offset.ll | 2108 +++++++++++++++++++++++
 llvm/test/CodeGen/AArch64/zext-to-tbl.ll        |   30 +-
 5 files changed, 2231 insertions(+), 55 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/vector-ldst-offset.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index 009d69b2b943..da617b7e1926 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -1583,6 +1583,8 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
   EVT DstVT = N->getValueType(0);
   ISD::MemIndexedMode AM = LD->getAddressingMode();
   bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
+  ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
+  int OffsetVal = (int)OffsetOp->getZExtValue();
 
   // We're not doing validity checking here. That was done when checking
   // if we should mark the load as indexed or not. We're just selecting
@@ -1637,18 +1639,58 @@ bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
     Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
   } else if (VT == MVT::f32) {
     Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
-  } else if (VT == MVT::f64 || VT.is64BitVector()) {
+  } else if (VT == MVT::f64 ||
+             (VT.is64BitVector() && Subtarget->isLittleEndian())) {
     Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
-  } else if (VT.is128BitVector()) {
+  } else if (VT.is128BitVector() && Subtarget->isLittleEndian()) {
     Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
+  } else if (VT.is64BitVector()) {
+    if (IsPre || OffsetVal != 8)
+      return false;
+    switch (VT.getScalarSizeInBits()) {
+    case 8:
+      Opcode = AArch64::LD1Onev8b_POST;
+      break;
+    case 16:
+      Opcode = AArch64::LD1Onev4h_POST;
+      break;
+    case 32:
+      Opcode = AArch64::LD1Onev2s_POST;
+      break;
+    case 64:
+      Opcode = AArch64::LD1Onev1d_POST;
+      break;
+    default:
+      llvm_unreachable("Expected vector element to be a power of 2");
+    }
+  } else if (VT.is128BitVector()) {
+    if (IsPre || OffsetVal != 16)
+      return false;
+    switch (VT.getScalarSizeInBits()) {
+    case 8:
+      Opcode = AArch64::LD1Onev16b_POST;
+      break;
+    case 16:
+      Opcode = AArch64::LD1Onev8h_POST;
+      break;
+    case 32:
+      Opcode = AArch64::LD1Onev4s_POST;
+      break;
+    case 64:
+      Opcode = AArch64::LD1Onev2d_POST;
+      break;
+    default:
+      llvm_unreachable("Expected vector element to be a power of 2");
+    }
   } else
     return false;
   SDValue Chain = LD->getChain();
   SDValue Base = LD->getBasePtr();
-  ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
-  int OffsetVal = (int)OffsetOp->getZExtValue();
   SDLoc dl(N);
-  SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
+  // LD1 encodes an immediate offset by using XZR as the offset register.
+  SDValue Offset = (VT.isVector() && !Subtarget->isLittleEndian())
+                       ? CurDAG->getRegister(AArch64::XZR, MVT::i64)
+                       : CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
   SDValue Ops[] = { Base, Offset, Chain };
   SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
                                        MVT::Other, Ops);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1169efce3123..22c497d3de64 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -2087,12 +2087,18 @@ void AArch64TargetLowering::addTypeForNEON(MVT VT) {
   setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
   setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
 
+  // When little-endian we can use ordinary d and q register loads/stores for
+  // vector types, but when big-endian we need to use structure load/store which
+  // only allow post-index addressing.
   if (Subtarget->isLittleEndian()) {
     for (unsigned im = (unsigned)ISD::PRE_INC;
          im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
       setIndexedLoadAction(im, VT, Legal);
       setIndexedStoreAction(im, VT, Legal);
     }
+  } else {
+    setIndexedLoadAction(ISD::POST_INC, VT, Legal);
+    setIndexedStoreAction(ISD::POST_INC, VT, Legal);
   }
 
   if (Subtarget->hasD128()) {
@@ -27047,6 +27053,12 @@ bool AArch64TargetLowering::getIndexedAddressParts(SDNode *N, SDNode *Op,
       RHSC = -(uint64_t)RHSC;
     if (!isInt<9>(RHSC))
       return false;
+    // When big-endian VLD1/VST1 are used for vector load and store, and these
+    // only allow an offset that's equal to the store size.
+    EVT MemType = cast<MemSDNode>(N)->getMemoryVT();
+    if (!Subtarget->isLittleEndian() && MemType.isVector() &&
+        RHSC != MemType.getStoreSize())
+      return false;
     // Always emit pre-inc/post-inc addressing mode. Use negated constant offset
     // when dealing with subtraction.
     Offset = DAG.getConstant(RHSC, SDLoc(N), RHS->getValueType(0));
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index f90f12b5ac3c..400ffff5d567 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4942,39 +4942,42 @@ def : Pat<(post_truncsti8 GPR64:$Rt, GPR64sp:$addr, simm9:$off),
 def : Pat<(post_store (bf16 FPR16:$Rt), GPR64sp:$addr, simm9:$off),
           (STRHpost FPR16:$Rt, GPR64sp:$addr, simm9:$off)>;
 
-def : Pat<(post_store (v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v4bf16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
-
-def : Pat<(post_store (v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
-def : Pat<(post_store (v8bf16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
-          (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+let Predicates = [IsLE] in {
+  // We must use ST1 to store vectors in big-endian.
+  def : Pat<(post_store(v8i8 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v4i16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v2i32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v2f32 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v1i64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v1f64 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v4f16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v4bf16 FPR64:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRDpost FPR64:$Rt, GPR64sp:$addr, simm9:$off)>;
+
+  def : Pat<(post_store(v16i8 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v8i16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v4i32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v4f32 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v2i64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v2f64 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v8f16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+  def : Pat<(post_store(v8bf16 FPR128:$Rt), GPR64sp:$addr, simm9:$off),
+            (STRQpost FPR128:$Rt, GPR64sp:$addr, simm9:$off)>;
+}
 
 //===----------------------------------------------------------------------===//
 // Load/store exclusive instructions.
@@ -8925,6 +8928,21 @@ def : St1Pat<v4i16, ST1Onev4h>;
 def : St1Pat<v2i32, ST1Onev2s>;
 def : St1Pat<v1i64, ST1Onev1d>;
 
+class St1PostPat<ValueType ty, Instruction INST, int off>
+    : Pat<(post_store ty:$Vt, GPR64sp:$Rn, (i64 off)),
+          (INST ty:$Vt, GPR64sp:$Rn, XZR)>;
+
+let Predicates = [IsBE] in {
+  def : St1PostPat<v16i8, ST1Onev16b_POST, 16>;
+  def : St1PostPat<v8i16, ST1Onev8h_POST, 16>;
+  def : St1PostPat<v4i32, ST1Onev4s_POST, 16>;
+  def : St1PostPat<v2i64, ST1Onev2d_POST, 16>;
+  def : St1PostPat<v8i8, ST1Onev8b_POST, 8>;
+  def : St1PostPat<v4i16, ST1Onev4h_POST, 8>;
+  def : St1PostPat<v2i32, ST1Onev2s_POST, 8>;
+  def : St1PostPat<v1i64, ST1Onev1d_POST, 8>;
+}
+
 //---
 // Single-element
 //---
diff --git a/llvm/test/CodeGen/AArch64/vector-ldst-offset.ll b/llvm/test/CodeGen/AArch64/vector-ldst-offset.ll
new file mode 100644
index 000000000000..b31ba46893bd
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vector-ldst-offset.ll
@@ -0,0 +1,2108 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 < %s -o - | FileCheck %s --check-prefixes=CHECK-LE
+; RUN: llc -mtriple=aarch64_be < %s -o - | FileCheck %s --check-prefixes=CHECK-BE
+
+; Check that we use the correct offset mode for vector loads and stores, and in
+; particular for big-endian we use ld1/st1 which only allows postindex immediate
+; offset of the same size as the memory access size.
+; FIXME: Currently we fail to make use of postindex register offset ld1/st1.
+
+define [2 x ptr] @v8i8_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i8_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i8>, ptr %ldptr, align 2
+  store <8 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i8_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i8_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #8
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <8 x i8>, ptr %add.ldptr, align 2
+  store <8 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i8_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i8_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i8>, ptr %ldptr, align 2
+  store <8 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i8_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i8_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <8 x i8>, ptr %add.ldptr, align 2
+  store <8 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i8_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8i8_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i8>, ptr %ldptr, align 2
+  store <8 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i8_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8i8_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i8_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.8b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <8 x i8>, ptr %add.ldptr, align 2
+  store <8 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i16_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i16>, ptr %ldptr, align 2
+  store <4 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i16_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #8
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <4 x i16>, ptr %add.ldptr, align 2
+  store <4 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i16_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i16>, ptr %ldptr, align 2
+  store <4 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i16_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <4 x i16>, ptr %add.ldptr, align 2
+  store <4 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4i16_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i16>, ptr %ldptr, align 2
+  store <4 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i16_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4i16_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i16_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <4 x i16>, ptr %add.ldptr, align 2
+  store <4 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i32_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i32>, ptr %ldptr, align 2
+  store <2 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i32_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #8
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <2 x i32>, ptr %add.ldptr, align 2
+  store <2 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i32_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i32>, ptr %ldptr, align 2
+  store <2 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i32_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <2 x i32>, ptr %add.ldptr, align 2
+  store <2 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2i32_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i32>, ptr %ldptr, align 2
+  store <2 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i32_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2i32_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i32_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <2 x i32>, ptr %add.ldptr, align 2
+  store <2 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1i64_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.1d }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.1d }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x i64>, ptr %ldptr, align 2
+  store <1 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1i64_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    str d0, [x8, #8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <1 x i64>, ptr %add.ldptr, align 2
+  store <1 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1i64_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0], #16
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    str d0, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x i64>, ptr %ldptr, align 2
+  store <1 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1i64_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    str d0, [x8, #16]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <1 x i64>, ptr %add.ldptr, align 2
+  store <1 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v1i64_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    str d0, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x i64>, ptr %ldptr, align 2
+  store <1 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1i64_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v1i64_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1i64_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    ldr d0, [x0, x2]
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    str d0, [x8, x2]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <1 x i64>, ptr %add.ldptr, align 2
+  store <1 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f16_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x half>, ptr %ldptr, align 2
+  store <4 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f16_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #8
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <4 x half>, ptr %add.ldptr, align 2
+  store <4 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f16_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x half>, ptr %ldptr, align 2
+  store <4 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f16_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <4 x half>, ptr %add.ldptr, align 2
+  store <4 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4f16_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x half>, ptr %ldptr, align 2
+  store <4 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f16_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4f16_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f16_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.4h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <4 x half>, ptr %add.ldptr, align 2
+  store <4 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f32_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x float>, ptr %ldptr, align 2
+  store <2 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f32_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #8
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <2 x float>, ptr %add.ldptr, align 2
+  store <2 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f32_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x float>, ptr %ldptr, align 2
+  store <2 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f32_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <2 x float>, ptr %add.ldptr, align 2
+  store <2 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2f32_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x float>, ptr %ldptr, align 2
+  store <2 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f32_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2f32_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f32_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.2s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <2 x float>, ptr %add.ldptr, align 2
+  store <2 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1f64_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #8
+; CHECK-LE-NEXT:    str d0, [x1], #8
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.1d }, [x0], #8
+; CHECK-BE-NEXT:    st1 { v0.1d }, [x1], #8
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x double>, ptr %ldptr, align 2
+  store <1 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1f64_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-LE-NEXT:    str d0, [x1, #8]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0, #8]!
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #8
+; CHECK-BE-NEXT:    str d0, [x8, #8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 8
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 8
+  %val = load <1 x double>, ptr %add.ldptr, align 2
+  store <1 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1f64_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0], #16
+; CHECK-LE-NEXT:    str d0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0], #16
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    str d0, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x double>, ptr %ldptr, align 2
+  store <1 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v1f64_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-LE-NEXT:    str d0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0, #16]!
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    str d0, [x8, #16]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <1 x double>, ptr %add.ldptr, align 2
+  store <1 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v1f64_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr d0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ldr d0, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    str d0, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <1 x double>, ptr %ldptr, align 2
+  store <1 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v1f64_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v1f64_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr d0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str d0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v1f64_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    ldr d0, [x0, x2]
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    str d0, [x8, x2]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <1 x double>, ptr %add.ldptr, align 2
+  store <1 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v16i8_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <16 x i8>, ptr %ldptr, align 2
+  store <16 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v16i8_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <16 x i8>, ptr %add.ldptr, align 2
+  store <16 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v16i8_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <16 x i8>, ptr %ldptr, align 2
+  store <16 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v16i8_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <16 x i8>, ptr %add.ldptr, align 2
+  store <16 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v16i8_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <16 x i8>, ptr %ldptr, align 2
+  store <16 x i8> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v16i8_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v16i8_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v16i8_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.16b }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <16 x i8>, ptr %add.ldptr, align 2
+  store <16 x i8> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i16_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i16>, ptr %ldptr, align 2
+  store <8 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i16_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <8 x i16>, ptr %add.ldptr, align 2
+  store <8 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i16_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i16>, ptr %ldptr, align 2
+  store <8 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8i16_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <8 x i16>, ptr %add.ldptr, align 2
+  store <8 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8i16_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x i16>, ptr %ldptr, align 2
+  store <8 x i16> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8i16_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8i16_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8i16_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <8 x i16>, ptr %add.ldptr, align 2
+  store <8 x i16> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i32_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i32>, ptr %ldptr, align 2
+  store <4 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i32_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <4 x i32>, ptr %add.ldptr, align 2
+  store <4 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i32_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i32>, ptr %ldptr, align 2
+  store <4 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4i32_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <4 x i32>, ptr %add.ldptr, align 2
+  store <4 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4i32_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x i32>, ptr %ldptr, align 2
+  store <4 x i32> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4i32_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4i32_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4i32_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <4 x i32>, ptr %add.ldptr, align 2
+  store <4 x i32> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i64_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i64>, ptr %ldptr, align 2
+  store <2 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i64_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <2 x i64>, ptr %add.ldptr, align 2
+  store <2 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i64_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i64>, ptr %ldptr, align 2
+  store <2 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2i64_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <2 x i64>, ptr %add.ldptr, align 2
+  store <2 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2i64_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x i64>, ptr %ldptr, align 2
+  store <2 x i64> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2i64_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2i64_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2i64_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <2 x i64>, ptr %add.ldptr, align 2
+  store <2 x i64> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8f16_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x half>, ptr %ldptr, align 2
+  store <8 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8f16_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <8 x half>, ptr %add.ldptr, align 2
+  store <8 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8f16_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x half>, ptr %ldptr, align 2
+  store <8 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v8f16_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <8 x half>, ptr %add.ldptr, align 2
+  store <8 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8f16_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <8 x half>, ptr %ldptr, align 2
+  store <8 x half> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v8f16_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v8f16_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v8f16_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.8h }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.8h }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <8 x half>, ptr %add.ldptr, align 2
+  store <8 x half> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f32_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x float>, ptr %ldptr, align 2
+  store <4 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f32_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <4 x float>, ptr %add.ldptr, align 2
+  store <4 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f32_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x float>, ptr %ldptr, align 2
+  store <4 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v4f32_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <4 x float>, ptr %add.ldptr, align 2
+  store <4 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4f32_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <4 x float>, ptr %ldptr, align 2
+  store <4 x float> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v4f32_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v4f32_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v4f32_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.4s }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.4s }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <4 x float>, ptr %add.ldptr, align 2
+  store <4 x float> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_postidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f64_postidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #16
+; CHECK-LE-NEXT:    str q0, [x1], #16
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_postidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0], #16
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1], #16
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x double>, ptr %ldptr, align 2
+  store <2 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_preidx_same_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f64_preidx_same_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #16]!
+; CHECK-LE-NEXT:    str q0, [x1, #16]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_preidx_same_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #16
+; CHECK-BE-NEXT:    add x1, x1, #16
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 16
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 16
+  %val = load <2 x double>, ptr %add.ldptr, align 2
+  store <2 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_postidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f64_postidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0], #32
+; CHECK-LE-NEXT:    str q0, [x1], #32
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_postidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x double>, ptr %ldptr, align 2
+  store <2 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_preidx_different_size(ptr %ldptr, ptr %stptr) {
+; CHECK-LE-LABEL: v2f64_preidx_different_size:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0, #32]!
+; CHECK-LE-NEXT:    str q0, [x1, #32]!
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_preidx_different_size:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, #32
+; CHECK-BE-NEXT:    add x1, x1, #32
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 32
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 32
+  %val = load <2 x double>, ptr %add.ldptr, align 2
+  store <2 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_postidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2f64_postidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    ldr q0, [x0]
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_postidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    mov x8, x1
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x8]
+; CHECK-BE-NEXT:    ret
+entry:
+  %val = load <2 x double>, ptr %ldptr, align 2
+  store <2 x double> %val, ptr %stptr, align 2
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
+
+define [2 x ptr] @v2f64_preidx_reg(ptr %ldptr, ptr %stptr, i64 %off) {
+; CHECK-LE-LABEL: v2f64_preidx_reg:
+; CHECK-LE:       // %bb.0: // %entry
+; CHECK-LE-NEXT:    mov x8, x1
+; CHECK-LE-NEXT:    ldr q0, [x0, x2]
+; CHECK-LE-NEXT:    add x0, x0, x2
+; CHECK-LE-NEXT:    add x1, x1, x2
+; CHECK-LE-NEXT:    str q0, [x8, x2]
+; CHECK-LE-NEXT:    ret
+;
+; CHECK-BE-LABEL: v2f64_preidx_reg:
+; CHECK-BE:       // %bb.0: // %entry
+; CHECK-BE-NEXT:    add x0, x0, x2
+; CHECK-BE-NEXT:    add x1, x1, x2
+; CHECK-BE-NEXT:    ld1 { v0.2d }, [x0]
+; CHECK-BE-NEXT:    st1 { v0.2d }, [x1]
+; CHECK-BE-NEXT:    ret
+entry:
+  %add.ldptr = getelementptr inbounds nuw i8, ptr %ldptr, i64 %off
+  %add.stptr = getelementptr inbounds nuw i8, ptr %stptr, i64 %off
+  %val = load <2 x double>, ptr %add.ldptr, align 2
+  store <2 x double> %val, ptr %add.stptr, align 2
+  %ret1 = insertvalue [2 x ptr] poison, ptr %add.ldptr, 0
+  %ret2 = insertvalue [2 x ptr] %ret1, ptr %add.stptr, 1
+  ret [2 x ptr] %ret2
+}
diff --git a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
index 68a9dff81232..2a37183c47d5 100644
--- a/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/zext-to-tbl.ll
@@ -2835,14 +2835,13 @@ define i32 @test_widening_instr_mull(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE:       // %bb.0: // %entry
 ; CHECK-BE-NEXT:  .LBB24_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1]
-; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
 ; CHECK-BE-NEXT:    add x8, x0, #16
+; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
 ; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
 ; CHECK-BE-NEXT:    add x9, x0, #48
 ; CHECK-BE-NEXT:    add x10, x0, #32
 ; CHECK-BE-NEXT:    subs w2, w2, #1
-; CHECK-BE-NEXT:    add x1, x1, #16
 ; CHECK-BE-NEXT:    ushll v2.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    ushll2 v0.8h, v0.16b, #0
 ; CHECK-BE-NEXT:    umull v4.4s, v1.4h, v2.4h
@@ -3094,7 +3093,7 @@ define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE-NEXT:    ld1 { v3.16b }, [x8]
 ; CHECK-BE-NEXT:  .LBB26_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v4.16b }, [x1]
+; CHECK-BE-NEXT:    ld1 { v4.16b }, [x1], #16
 ; CHECK-BE-NEXT:    add x8, x0, #32
 ; CHECK-BE-NEXT:    ld1 { v16.4s }, [x0]
 ; CHECK-BE-NEXT:    add x9, x0, #48
@@ -3107,7 +3106,6 @@ define i32 @test_widening_instr_mull_2(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE-NEXT:    tbl v6.16b, { v4.16b }, v3.16b
 ; CHECK-BE-NEXT:    tbl v7.16b, { v4.16b }, v2.16b
 ; CHECK-BE-NEXT:    tbl v4.16b, { v4.16b }, v0.16b
-; CHECK-BE-NEXT:    add x1, x1, #16
 ; CHECK-BE-NEXT:    rev32 v5.16b, v5.16b
 ; CHECK-BE-NEXT:    rev32 v6.16b, v6.16b
 ; CHECK-BE-NEXT:    rev32 v7.16b, v7.16b
@@ -3175,19 +3173,18 @@ define i32 @mul_zext_16i8_sext_16i8(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE:       // %bb.0: // %entry
 ; CHECK-BE-NEXT:  .LBB27_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x0]
-; CHECK-BE-NEXT:    ld1 { v1.16b }, [x1]
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
 ; CHECK-BE-NEXT:    add x8, x0, #48
+; CHECK-BE-NEXT:    ld1 { v1.16b }, [x0]
 ; CHECK-BE-NEXT:    subs w2, w2, #1
-; CHECK-BE-NEXT:    add x1, x1, #16
-; CHECK-BE-NEXT:    sshll2 v2.8h, v0.16b, #0
-; CHECK-BE-NEXT:    ushll2 v3.8h, v1.16b, #0
-; CHECK-BE-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-BE-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-BE-NEXT:    sshll2 v2.8h, v1.16b, #0
+; CHECK-BE-NEXT:    ushll2 v3.8h, v0.16b, #0
+; CHECK-BE-NEXT:    sshll v1.8h, v1.8b, #0
+; CHECK-BE-NEXT:    ushll v0.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    smull2 v4.4s, v2.8h, v3.8h
 ; CHECK-BE-NEXT:    smull v2.4s, v2.4h, v3.4h
-; CHECK-BE-NEXT:    smull v3.4s, v0.4h, v1.4h
-; CHECK-BE-NEXT:    smull2 v0.4s, v0.8h, v1.8h
+; CHECK-BE-NEXT:    smull v3.4s, v1.4h, v0.4h
+; CHECK-BE-NEXT:    smull2 v0.4s, v1.8h, v0.8h
 ; CHECK-BE-NEXT:    st1 { v4.4s }, [x8]
 ; CHECK-BE-NEXT:    add x8, x0, #32
 ; CHECK-BE-NEXT:    st1 { v3.4s }, [x0]
@@ -3249,14 +3246,13 @@ define i32 @mul_zext_16i8_sext_16i16(ptr %p1, ptr %p2, i32 %h) {
 ; CHECK-BE:       // %bb.0: // %entry
 ; CHECK-BE-NEXT:  .LBB28_1: // %loop
 ; CHECK-BE-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1]
-; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
+; CHECK-BE-NEXT:    ld1 { v0.16b }, [x1], #16
 ; CHECK-BE-NEXT:    add x8, x0, #16
+; CHECK-BE-NEXT:    ld1 { v1.8h }, [x0]
 ; CHECK-BE-NEXT:    ld1 { v3.8h }, [x8]
 ; CHECK-BE-NEXT:    add x9, x0, #48
 ; CHECK-BE-NEXT:    add x10, x0, #32
 ; CHECK-BE-NEXT:    subs w2, w2, #1
-; CHECK-BE-NEXT:    add x1, x1, #16
 ; CHECK-BE-NEXT:    ushll v2.8h, v0.8b, #0
 ; CHECK-BE-NEXT:    ushll2 v0.8h, v0.16b, #0
 ; CHECK-BE-NEXT:    smull v4.4s, v1.4h, v2.4h
-- 
cgit v1.2.3


From 3af4d4e8100fda2a7e1bd0dbbe0914b584ad08d6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nathan=20Gau=C3=ABr?= <brioche@google.com>
Date: Wed, 18 Jun 2025 17:26:40 +0200
Subject: [HLSL][SPIR-V] Fix LinkageAttribute emission for BuiltIn (#144701)

BuiltIn variables were missing the visibility attribute, which caused
the Linkage capability to be emitted by the backend.
---
 clang/lib/CodeGen/CGHLSLRuntime.cpp                |  1 +
 .../test/CodeGenHLSL/semantics/SV_Position.ps.hlsl |  2 +-
 .../CodeGen/SPIRV/linkage/link-attribute-vk.ll     | 23 ++++++++++++++++++++++
 3 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/SPIRV/linkage/link-attribute-vk.ll

diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 585411bc59e1..34960c34e109 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -375,6 +375,7 @@ static llvm::Value *createSPIRVBuiltinLoad(IRBuilder<> &B, llvm::Module &M,
       llvm::GlobalVariable::GeneralDynamicTLSModel,
       /* AddressSpace */ 7, /* isExternallyInitialized= */ true);
   addSPIRVBuiltinDecoration(GV, BuiltInID);
+  GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
   return B.CreateLoad(Ty, GV);
 }
 
diff --git a/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl b/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl
index 58b91fc9264d..bdba38e028ed 100644
--- a/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl
+++ b/clang/test/CodeGenHLSL/semantics/SV_Position.ps.hlsl
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-pixel -x hlsl -emit-llvm -finclude-default-header -disable-llvm-passes -o - %s | FileCheck %s
 
-// CHECK: @sv_position = external thread_local addrspace(7) externally_initialized constant <4 x float>, !spirv.Decorations !0
+// CHECK: @sv_position = external hidden thread_local addrspace(7) externally_initialized constant <4 x float>, !spirv.Decorations !0
 
 // CHECK: define void @main() {{.*}} {
 float4 main(float4 p : SV_Position) {
diff --git a/llvm/test/CodeGen/SPIRV/linkage/link-attribute-vk.ll b/llvm/test/CodeGen/SPIRV/linkage/link-attribute-vk.ll
new file mode 100644
index 000000000000..d4ba61ff58d3
--- /dev/null
+++ b/llvm/test/CodeGen/SPIRV/linkage/link-attribute-vk.ll
@@ -0,0 +1,23 @@
+; RUN: llc -verify-machineinstrs -O0 -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - | FileCheck %s
+; RUN: %if spirv-tools %{ llc -O0 -mtriple=spirv-unknown-vulkan1.3-pixel %s -o - -filetype=obj | spirv-val --target-env vulkan1.3 %}
+
+@sv_position = external hidden thread_local local_unnamed_addr addrspace(7) externally_initialized constant <4 x float>, !spirv.Decorations !0
+
+; CHECK-NOT: OpDecorate %[[#var]] LinkageAttributes "sv_position" Import
+
+; CHECK-DAG: %[[#float:]] = OpTypeFloat 32
+; CHECK-DAG: %[[#float4:]] = OpTypeVector %[[#float]]
+; CHECK-DAG: %[[#type:]] = OpTypePointer Input %[[#float4]]
+; CHECK-DAG: %[[#var:]] = OpVariable %[[#type]] Input
+
+; CHECK-NOT: OpDecorate %[[#var]] LinkageAttributes "sv_position" Import
+
+define void @main() #1 {
+entry:
+  ret void
+}
+
+attributes #1 = { "hlsl.shader"="pixel" }
+
+!0 = !{!1}
+!1 = !{i32 11, i32 0}
-- 
cgit v1.2.3


From 8b8a3699dbdbb5d7865b0fe330d972c3fa380f1e Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Wed, 18 Jun 2025 16:27:29 +0100
Subject: [AArch64] Use dupq (SVE2.1) for segmented lane splats (#144482)

Use the dupq instructions (when available) to represent a splat of the
same lane within each 128b segment of a wider fixed vector.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    |  37 +++++++
 .../test/CodeGen/AArch64/sve2p1-vector-shuffles.ll | 115 +++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 22c497d3de64..0e28ccd0f655 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -13392,6 +13392,30 @@ static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
   return true;
 }
 
+/// isDUPQMask - matches a splat of equivalent lanes within 128b segments in
+/// the first vector operand.
+static std::optional<unsigned> isDUPQMask(ArrayRef<int> M, EVT VT) {
+  assert(VT.getFixedSizeInBits() % 128 == 0 && "Unsupported SVE vector size");
+  unsigned Lane = (unsigned)M[0];
+  unsigned Segments = VT.getFixedSizeInBits() / 128;
+  unsigned SegmentElts = VT.getVectorNumElements() / Segments;
+
+  // Make sure there's no size changes.
+  if (SegmentElts * Segments != M.size())
+    return std::nullopt;
+
+  // Check the first index corresponds to one of the lanes in the first segment.
+  if (Lane >= SegmentElts)
+    return std::nullopt;
+
+  // Check that all lanes match the first, adjusted for segment.
+  for (unsigned I = 0; I < M.size(); ++I)
+    if ((unsigned)M[I] != (Lane + ((I / SegmentElts) * SegmentElts)))
+      return std::nullopt;
+
+  return Lane;
+}
+
 /// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
 /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
 /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
@@ -29981,6 +30005,19 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE(
       return convertFromScalableVector(
           DAG, VT, DAG.getNode(Opc, DL, ContainerVT, Op1, Op1));
     }
+
+    if (Subtarget->hasSVE2p1()) {
+      if (std::optional<unsigned> Lane = isDUPQMask(ShuffleMask, VT)) {
+        SDValue IID =
+            DAG.getConstant(Intrinsic::aarch64_sve_dup_laneq, DL, MVT::i64);
+        return convertFromScalableVector(
+            DAG, VT,
+            DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
+                        {IID, Op1,
+                         DAG.getConstant(*Lane, DL, MVT::i64,
+                                         /*isTarget=*/true)}));
+      }
+    }
   }
 
   // Try to widen the shuffle before generating a possibly expensive SVE TBL.
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll b/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll
new file mode 100644
index 000000000000..40d4d0ff6014
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-vector-shuffles.ll
@@ -0,0 +1,115 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+define void @dupq_i8_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_i8_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.b, z0.b[15]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <32 x i8>, ptr %addr
+  %splat.lanes = shufflevector <32 x i8> %load, <32 x i8> poison, <32 x i32> <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15,
+                                                                              i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31, i32 31>
+  store <32 x i8> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_i16_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_i16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.h, z0.h[2]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <16 x i16>, ptr %addr
+  %splat.lanes = shufflevector <16 x i16> %load, <16 x i16> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  store <16 x i16> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_i32_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_i32_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.s, z0.s[3]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <8 x i32>, ptr %addr
+  %splat.lanes = shufflevector <8 x i32> %load, <8 x i32> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                             i32 7, i32 7, i32 7, i32 7>
+  store <8 x i32> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_i64_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_i64_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    trn1 z0.d, z0.d, z0.d
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <4 x i64>, ptr %addr
+  %splat.lanes = shufflevector <4 x i64> %load, <4 x i64> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  store <4 x i64> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_f16_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_f16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.h, z0.h[2]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <16 x half>, ptr %addr
+  %splat.lanes = shufflevector <16 x half> %load, <16 x half> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                  i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  store <16 x half> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_bf16_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_bf16_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldp q0, q1, [x0]
+; CHECK-NEXT:    dup v0.8h, v0.h[2]
+; CHECK-NEXT:    dup v1.8h, v1.h[2]
+; CHECK-NEXT:    stp q0, q1, [x0]
+; CHECK-NEXT:    ret
+  %load = load <16 x bfloat>, ptr %addr
+  %splat.lanes = shufflevector <16 x bfloat> %load, <16 x bfloat> poison, <16 x i32> <i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2, i32  2,
+                                                                                      i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10, i32 10>
+  store <16 x bfloat> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_f32_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_f32_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    dupq z0.s, z0.s[3]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <8 x float>, ptr %addr
+  %splat.lanes = shufflevector <8 x float> %load, <8 x float> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3,
+                                                                                 i32 7, i32 7, i32 7, i32 7>
+  store <8 x float> %splat.lanes, ptr %addr
+  ret void
+}
+
+define void @dupq_f64_256b(ptr %addr) #0 {
+; CHECK-LABEL: dupq_f64_256b:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ldr z0, [x0]
+; CHECK-NEXT:    trn1 z0.d, z0.d, z0.d
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %load = load <4 x double>, ptr %addr
+  %splat.lanes = shufflevector <4 x double> %load, <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 2, i32 2>
+  store <4 x double> %splat.lanes, ptr %addr
+  ret void
+}
+
+attributes #0 = { noinline vscale_range(2,2) "target-features"="+sve2p1,+bf16" }
-- 
cgit v1.2.3


From 9da9d32670ddbf610f0788236e78b2382037f00b Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Wed, 18 Jun 2025 11:30:53 -0400
Subject: [AMDGPU][True16][CodeGen] sext i16 inreg in true16 mode (#144024)

update sext pattern in true16, setting up proper vgpr16 reg use
---
 llvm/lib/Target/AMDGPU/SIInstructions.td           |  19 ++
 llvm/lib/Target/AMDGPU/VOP3Instructions.td         |  24 ++
 llvm/test/CodeGen/AMDGPU/idot4s.ll                 |  69 +++---
 llvm/test/CodeGen/AMDGPU/idot4u.ll                 |  79 +++----
 llvm/test/CodeGen/AMDGPU/llvm.frexp.ll             |  64 ++---
 .../CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll   |  86 +++++++
 llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll     | 260 ++++++++++++---------
 llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll     | 260 ++++++++++++---------
 8 files changed, 521 insertions(+), 340 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll

diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 897c30948cf0..56b15c11a669 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -2623,6 +2623,8 @@ def : GCNPat<
   (i32 (DivergentSextInreg<i1> i32:$src)),
   (V_BFE_I32_e64 i32:$src, (i32 0), (i32 1))>;
 
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in {
 def : GCNPat <
   (i16 (DivergentSextInreg<i1> i16:$src)),
   (V_BFE_I32_e64 $src, (i32 0), (i32 1))
@@ -2632,6 +2634,23 @@ def : GCNPat <
   (i16 (DivergentSextInreg<i8> i16:$src)),
   (V_BFE_I32_e64 $src, (i32 0), (i32 8))
 >;
+}
+
+let True16Predicate = UseRealTrue16Insts in {
+def : GCNPat <
+  (i16 (DivergentSextInreg<i1> i16:$src)),
+  (V_BFE_I32_e64
+   (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+   (i32 0), (i32 1))
+>;
+
+def : GCNPat <
+  (i16 (DivergentSextInreg<i8> i16:$src)),
+  (V_BFE_I32_e64
+   (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+   (i32 0), (i32 8))
+>;
+}
 
 def : GCNPat<
   (i32 (DivergentSextInreg<i8> i32:$src)),
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 2dbc119f65cd..89a9ecc27c6e 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -319,11 +319,21 @@ let SchedRW = [Write64Bit] in {
 } // End SchedRW = [Write64Bit]
 } // End isReMaterializable = 1
 
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in
 def : GCNPat<
   (i32 (DivergentUnaryFrag<sext> i16:$src)),
   (i32 (V_BFE_I32_e64 i16:$src, (i32 0), (i32 0x10)))
 >;
 
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat<
+  (i32 (DivergentUnaryFrag<sext> i16:$src)),
+  (i32 (V_BFE_I32_e64
+       (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+       (i32 0), (i32 0x10)))
+>;
+
 let isReMaterializable = 1 in {
 let SubtargetPredicate = isGFX6GFX7GFX10Plus in {
 defm V_MULLIT_F32 : VOP3Inst <"v_mullit_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
@@ -423,6 +433,8 @@ def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32
 
 } // End SubtargetPredicate = Has16BitInsts, isCommutable = 1
 
+foreach p = [NotHasTrue16BitInsts, UseFakeTrue16Insts] in
+let True16Predicate = p in
 def : GCNPat<
   (i64 (DivergentUnaryFrag<sext> i16:$src)),
     (REG_SEQUENCE VReg_64,
@@ -432,6 +444,18 @@ def : GCNPat<
       ), VGPR_32)), sub1)
 >;
 
+let True16Predicate = UseRealTrue16Insts in
+def : GCNPat<
+  (i64 (DivergentUnaryFrag<sext> i16:$src)),
+    (REG_SEQUENCE VReg_64,
+      (i32 (V_BFE_I32_e64
+            (REG_SEQUENCE VGPR_32, VGPR_16:$src, lo16, (i16 (IMPLICIT_DEF)), hi16),
+            (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10)))), sub0,
+      (i32 (COPY_TO_REGCLASS
+         (V_ASHRREV_I32_e32 (S_MOV_B32 (i32 0x1f)), (i32 (V_BFE_I32_e64 $src, (S_MOV_B32 (i32 0)), (S_MOV_B32 (i32 0x10))))
+      ), VGPR_32)), sub1)
+>;
+
 let SubtargetPredicate = isGFX8Plus, Uses = [MODE, M0, EXEC], OtherPredicates = [isNotGFX90APlus] in {
 def V_INTERP_P1_F32_e64  : VOP3Interp <"v_interp_p1_f32", VOP3_INTERP>;
 def V_INTERP_P2_F32_e64  : VOP3Interp <"v_interp_p2_f32", VOP3_INTERP>;
diff --git a/llvm/test/CodeGen/AMDGPU/idot4s.ll b/llvm/test/CodeGen/AMDGPU/idot4s.ll
index 9e7968f1acb8..ab38bd21994e 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4s.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4s.ll
@@ -1165,35 +1165,32 @@ define amdgpu_kernel void @idot4_acc16_vecMul(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    s_clause 0x1
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v1, v0, s[2:3]
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v1, v0, s[0:1]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v2, v0, s[2:3]
 ; GFX11-DL-TRUE16-NEXT:    global_load_d16_b16 v0, v3, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.l
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v6.h, 8, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.h
-; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v8.h, 8, v1.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v1.h
-; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v2.h, 8, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v2, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v4.h, 8, v1.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v1.h
+; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v7.h, 8, v2.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v5.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v1.h, 8, v1.h
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v8.l, v4.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v5.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v9, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v7, 0, 8
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_pk_mul_lo_u16 v6, v6, v8
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v2, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_ashrrev_i16 v2.h, 8, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_pk_mul_lo_u16 v4, v4, v7
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v6.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.l
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v6.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v4.l, v0.l
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-DL-TRUE16-NEXT:    v_pk_mul_lo_u16 v1, v2, v1
-; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v6.h
+; GFX11-DL-TRUE16-NEXT:    v_pk_mul_lo_u16 v1, v1, v2
+; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v4.h
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.l
 ; GFX11-DL-TRUE16-NEXT:    v_add_nc_u16 v0.l, v0.l, v1.h
@@ -3435,35 +3432,31 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-NEXT:    global_load_b32 v2, v0, s[0:1]
 ; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[2:3]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v3
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v2.h
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v0, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v1.l
 ; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v6.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.h
+; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v5.l
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v5, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v7, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 24, v2
 ; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v1.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v2.l
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.h, v2.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v3.l, v0.l
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
 ; GFX11-DL-TRUE16-NEXT:    global_store_b32 v1, v0, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index f995f426c637..5e502882a264 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -1669,40 +1669,38 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes:
 ; GFX11-DL-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-DL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT:    v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-DL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
 ; GFX11-DL-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-TRUE16-NEXT:    v_mov_b32_e32 v6, 0
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    s_clause 0x1
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[0:1]
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v4, v0, s[2:3]
-; GFX11-DL-TRUE16-NEXT:    global_load_d16_b16 v0, v5, s[4:5]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v4, v0, s[0:1]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v5, v0, s[2:3]
+; GFX11-DL-TRUE16-NEXT:    global_load_d16_b16 v0, v6, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v4
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 8, v4
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 8, v5
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v7, v5, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v1.l
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v6, v7, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v1.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v6.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v2.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v1, v4, v4, 0xc0c0302
-; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v2, v3, v3, 0xc0c0302
+; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v1, v5, v5, 0xc0c0302
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v2.l, v3.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_perm_b32 v2, v4, v4, 0xc0c0302
 ; GFX11-DL-TRUE16-NEXT:    v_and_b32_e32 v0, 0xffff, v0
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_dot4_u32_u8 v0, v2, v1, v0
-; GFX11-DL-TRUE16-NEXT:    global_store_b16 v5, v0, s[4:5]
+; GFX11-DL-TRUE16-NEXT:    global_store_b16 v6, v0, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes:
@@ -1964,44 +1962,41 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-LABEL: notdot4_mixedtypes2:
 ; GFX11-DL-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-DL-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-DL-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-DL-TRUE16-NEXT:    v_dual_mov_b32 v5, 0 :: v_dual_and_b32 v0, 0x3ff, v0
 ; GFX11-DL-TRUE16-NEXT:    s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-TRUE16-NEXT:    v_mov_b32_e32 v4, 0
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    s_clause 0x1
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v2, v0, s[2:3]
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[0:1]
-; GFX11-DL-TRUE16-NEXT:    global_load_d16_b16 v0, v4, s[4:5]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[2:3]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v4, v0, s[0:1]
+; GFX11-DL-TRUE16-NEXT:    global_load_d16_b16 v0, v5, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(2)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v2
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v3
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v3.l
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v3
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v3.h
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v2, 8, v4
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v6, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v4.h
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v6.l
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 24, v2
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v2.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.l
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.l
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v6.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 24, v3
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v1.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v3.h
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 24, v4
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v2.l, v1.h, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v2, v5, 0, 8
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v0.h, v0.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v3.l, v1.l, v0.l
-; GFX11-DL-TRUE16-NEXT:    global_store_b16 v4, v0, s[4:5]
+; GFX11-DL-TRUE16-NEXT:    global_store_b16 v5, v0, s[4:5]
 ; GFX11-DL-TRUE16-NEXT:    s_endpgm
 ;
 ; GFX11-DL-FAKE16-LABEL: notdot4_mixedtypes2:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
index f44faf4f7edb..3a4bf1c81ed5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
@@ -424,15 +424,15 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; GFX11-SDAG-TRUE16-LABEL: test_frexp_v2f16_v2i32:
 ; GFX11-SDAG-TRUE16:       ; %bb.0:
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 16
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX11-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v3.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_pack_b32_f16 v0, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 16
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 16
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-SDAG-FAKE16-LABEL: test_frexp_v2f16_v2i32:
@@ -457,15 +457,15 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.h, v0.h
-; GFX12-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 16
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX12-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v3.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_pack_b32_f16 v0, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 16
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 16
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-SDAG-FAKE16-LABEL: test_frexp_v2f16_v2i32:
@@ -534,15 +534,15 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; GFX11-GISEL-TRUE16-LABEL: test_frexp_v2f16_v2i32:
 ; GFX11-GISEL-TRUE16:       ; %bb.0:
 ; GFX11-GISEL-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.h
-; GFX11-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.h, v0.h
-; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 16
-; GFX11-GISEL-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX11-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.h, v0.h
+; GFX11-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v3.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-TRUE16-NEXT:    v_pack_b32_f16 v0, v1.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 16
 ; GFX11-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX11-GISEL-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.l, v0.h
+; GFX11-GISEL-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 16
 ; GFX11-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-GISEL-FAKE16-LABEL: test_frexp_v2f16_v2i32:
@@ -567,15 +567,15 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
 ; GFX12-GISEL-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-GISEL-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-GISEL-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.h
-; GFX12-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v0.h, v0.h
-; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 16
-; GFX12-GISEL-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 16
+; GFX12-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT:    v_frexp_mant_f16_e32 v1.h, v0.h
+; GFX12-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v2.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT:    v_frexp_exp_i16_f16_e32 v3.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-TRUE16-NEXT:    v_pack_b32_f16 v0, v1.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 16
 ; GFX12-GISEL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3)
-; GFX12-GISEL-TRUE16-NEXT:    v_pack_b32_f16 v0, v0.l, v0.h
+; GFX12-GISEL-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 16
 ; GFX12-GISEL-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX12-GISEL-FAKE16-LABEL: test_frexp_v2f16_v2i32:
diff --git a/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll b/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll
new file mode 100644
index 000000000000..49dec15f9f7d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/sext-in-reg-vector-shuffle.ll
@@ -0,0 +1,86 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN:  llc -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -enable-var-scope --check-prefixes=GFX11-TRUE16 %s
+; RUN:  llc -mtriple=amdgcn-- -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -enable-var-scope --check-prefixes=GFX11-FAKE16 %s
+
+define amdgpu_kernel void @v_sext_in_reg_i8_i16_shuffle_vector(ptr addrspace(1) %out, ptr addrspace(1) %ptr) #0 {
+;
+; GFX11-TRUE16-LABEL: v_sext_in_reg_i8_i16_shuffle_vector:
+; GFX11-TRUE16:       ; %bb.0:
+; GFX11-TRUE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-TRUE16-NEXT:    v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT:    global_load_b64 v[1:2], v0, s[2:3]
+; GFX11-TRUE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v2.h
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v1.h
+; GFX11-TRUE16-NEXT:    v_bfe_i32 v7, v2, 0, 8
+; GFX11-TRUE16-NEXT:    v_ashrrev_i32_e32 v5, 24, v2
+; GFX11-TRUE16-NEXT:    v_ashrrev_i32_e32 v6, 24, v1
+; GFX11-TRUE16-NEXT:    v_bfe_i32 v8, v3, 0, 8
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.l, 8, v1.l
+; GFX11-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX11-TRUE16-NEXT:    v_ashrrev_i16 v0.h, 8, v2.l
+; GFX11-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v7.l
+; GFX11-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v0.h, v0.h
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v2.h, v6.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT:    v_cvt_f16_i16_e32 v4.l, v4.l
+; GFX11-TRUE16-NEXT:    v_mov_b32_e32 v5, 0
+; GFX11-TRUE16-NEXT:    v_pack_b32_f16 v3, v0.l, v1.l
+; GFX11-TRUE16-NEXT:    v_pack_b32_f16 v1, v0.h, v1.h
+; GFX11-TRUE16-NEXT:    v_pack_b32_f16 v2, v2.h, v2.l
+; GFX11-TRUE16-NEXT:    v_pack_b32_f16 v0, v4.h, v4.l
+; GFX11-TRUE16-NEXT:    global_store_b128 v5, v[0:3], s[0:1]
+; GFX11-TRUE16-NEXT:    s_endpgm
+;
+; GFX11-FAKE16-LABEL: v_sext_in_reg_i8_i16_shuffle_vector:
+; GFX11-FAKE16:       ; %bb.0:
+; GFX11-FAKE16-NEXT:    s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FAKE16-NEXT:    v_dual_mov_b32 v9, 0 :: v_dual_and_b32 v0, 0x3ff, v0
+; GFX11-FAKE16-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT:    global_load_b64 v[0:1], v0, s[2:3]
+; GFX11-FAKE16-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT:    v_ashrrev_i32_e32 v2, 24, v1
+; GFX11-FAKE16-NEXT:    v_ashrrev_i32_e32 v5, 24, v0
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v6, 8, v1
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v7, v0, 0, 8
+; GFX11-FAKE16-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v3, v3, 0, 8
+; GFX11-FAKE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v7, v7
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v0, v0
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v1, v1
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v6, v6
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v5, v5
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v8, v2
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v2, v4
+; GFX11-FAKE16-NEXT:    v_cvt_f16_i16_e32 v4, v3
+; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v3, v0, v7
+; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v1, v6, v1
+; GFX11-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v2, v5, v2
+; GFX11-FAKE16-NEXT:    v_pack_b32_f16 v0, v8, v4
+; GFX11-FAKE16-NEXT:    global_store_b128 v9, v[0:3], s[0:1]
+; GFX11-FAKE16-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %in.gep = getelementptr <{ [0 x i8] }>, ptr addrspace(1) %ptr, i64 0, i32 0, i32 %tid
+  %load = load <8 x i8>, ptr addrspace(1) %in.gep
+  %shuff = shufflevector <8 x i8> %load, <8 x i8> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+  %cast = sitofp <8 x i8> %shuff to <8 x half>
+  store <8 x half> %cast, ptr addrspace(1) %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() #1
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 2d4c881b855e..16fbd1eabb30 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -208,11 +208,16 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v0.l, 0xff80
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -245,11 +250,16 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v0.l, 0xff80
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -790,7 +800,7 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v1, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v5, 0, 8
@@ -798,24 +808,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v5.l, v3.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v6, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -897,7 +910,7 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v1, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v5, 0, 8
@@ -905,24 +918,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v5.l, v3.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v6, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1291,51 +1307,59 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v16i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v11, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v17, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v15, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v16.l
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v1.h, v5.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v3.h, v9.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v15, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v14, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v3.l, v13.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v10, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v14, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v3.h, v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v2.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.h, v1.l, v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v9, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v17, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.h, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v16.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.h, v2.l, v3.l, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.h, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v0.l, v1.l, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
@@ -1444,51 +1468,59 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v11, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v17, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v15, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v16.l
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v1.h, v5.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v3.h, v9.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v15, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v14, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v3.l, v13.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v10, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v14, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v1.l, v3.h, v1.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v2.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.h, v1.l, v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v9, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.l, v6.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v17, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.h, v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.h, v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v16.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.l
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.h, v2.l, v3.l, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v1.h, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_max3_i16 v0.l, v0.l, v1.l, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_max_i16 v0.l, v0.l, v1.l
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index d9d9a6b9a4b1..bb868621c23d 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -208,11 +208,16 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v0.l, 0x7f
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -245,11 +250,16 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v0.l, 0x7f
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -790,7 +800,7 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v1, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v5, 0, 8
@@ -798,24 +808,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v5.l, v3.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v6, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v6
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v2.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -897,7 +910,7 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v1, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v5, 0, 8
@@ -905,24 +918,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v5.l, v3.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v6, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v6.l, 8, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v6
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v2.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1291,51 +1307,59 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
 ; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v16i8:
 ; GFX11-SDAG-TRUE16:       ; %bb.0: ; %entry
 ; GFX11-SDAG-TRUE16-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v11, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v17, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v15, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v16.l
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v1.h, v5.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v3.h, v9.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v15, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v14, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v3.l, v13.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v10, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v14, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v3.h, v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v2.l, v3.l
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.h, v1.l, v2.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v9, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v6.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v17, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.h, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v16.l
+; GFX11-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.l
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v5.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.h, v2.l, v3.l, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.h, v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v0.l, v1.l, v0.h
 ; GFX11-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX11-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
@@ -1444,51 +1468,59 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_samplecnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_bvhcnt 0x0
 ; GFX12-SDAG-TRUE16-NEXT:    s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v11, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v17, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v15, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v7.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v13.l, v16.l
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v1.h, v5.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v3.h, v9.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v16, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v17.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v11, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v15, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v3, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v14, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v9.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v13, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v3.l, v13.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v8, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v4.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v10, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v7, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v14, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v1.l, v3.h, v1.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v3, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v9.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v8, 8, v9
-; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v10.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v4.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v2.l, v3.l
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.h, v1.l, v2.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v11.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v9, v9, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.l, v6.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v6, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v5, v17, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.h, v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v4, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.h, v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v16.l
+; GFX12-SDAG-TRUE16-NEXT:    v_lshlrev_b16 v7.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v10.l
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v5.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.h, v2.l, v3.l, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v1.h, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT:    v_min3_i16 v0.l, v0.l, v1.l, v0.h
 ; GFX12-SDAG-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
 ; GFX12-SDAG-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX12-SDAG-TRUE16-NEXT:    v_min_i16 v0.l, v0.l, v1.l
-- 
cgit v1.2.3


From 5d502aeddf2a5d93c3fd93103054261acf4d92f3 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 18 Jun 2025 10:42:39 -0500
Subject: [flang][OpenMP] Clarify confusing error message (#144707)

The message "The atomic variable x should occur exactly once among the
arguments of the top-level [...] operator" was intended to convey that
(1) an atomic variable should be an argument, and (2) it should be
exactly one of the arguments. However, the wording turned out to be
sowing confusion instead.

Rework the corresponding check, and emit an individual error message for
each problematic situation:
- "atomic variable cannot be a proper subexpression of an argument",
- "atomic variable should appear as an argument",
- "atomic variable should be exactly one of the arguments".

Fixes https://github.com/llvm/llvm-project/issues/144599
---
 flang/lib/Semantics/check-omp-structure.cpp        | 57 +++++++++++++-------
 flang/test/Semantics/OpenMP/atomic-update-only.f90 |  3 +-
 flang/test/Semantics/OpenMP/atomic03.f90           | 32 ++++++------
 flang/test/Semantics/OpenMP/atomic04.f90           | 60 ++++++++++++----------
 4 files changed, 88 insertions(+), 64 deletions(-)

diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 83f4d1edf3c4..36d4bcb5d99f 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -3510,37 +3510,56 @@ void OmpStructureChecker::CheckAtomicUpdateAssignment(
         operation::ToString(top.first));
     return;
   }
-  // Check if `atom` occurs exactly once in the argument list.
+  // Check how many times `atom` occurs as an argument, if it's a subexpression
+  // of an argument, and collect the non-atom arguments.
   std::vector<SomeExpr> nonAtom;
-  auto unique{[&]() { // -> iterator
-    auto found{top.second.end()};
-    for (auto i{top.second.begin()}, e{top.second.end()}; i != e; ++i) {
-      if (IsSameOrConvertOf(*i, atom)) {
-        if (found != top.second.end()) {
-          return top.second.end();
-        }
-        found = i;
+  MaybeExpr subExpr;
+  auto atomCount{[&]() {
+    int count{0};
+    for (const SomeExpr &arg : top.second) {
+      if (IsSameOrConvertOf(arg, atom)) {
+        ++count;
       } else {
-        nonAtom.push_back(*i);
+        if (!subExpr && IsSubexpressionOf(atom, arg)) {
+          subExpr = arg;
+        }
+        nonAtom.push_back(arg);
       }
     }
-    return found;
+    return count;
   }()};
 
-  if (unique == top.second.end()) {
-    if (top.first == operation::Operator::Identity) {
-      // This is "x = y".
+  bool hasError{false};
+  if (subExpr) {
+    context_.Say(rsrc,
+        "The atomic variable %s cannot be a proper subexpression of an argument (here: %s) in the update operation"_err_en_US,
+        atom.AsFortran(), subExpr->AsFortran());
+    hasError = true;
+  }
+  if (top.first == operation::Operator::Identity) {
+    // This is "x = y".
+    assert((atomCount == 0 || atomCount == 1) && "Unexpected count");
+    if (atomCount == 0) {
       context_.Say(rsrc,
           "The atomic variable %s should appear as an argument in the update operation"_err_en_US,
           atom.AsFortran());
-    } else {
-      assert(top.first != operation::Operator::Identity &&
-          "Handle this separately");
+      hasError = true;
+    }
+  } else {
+    if (atomCount == 0) {
+      context_.Say(rsrc,
+          "The atomic variable %s should appear as an argument of the top-level %s operator"_err_en_US,
+          atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
+    } else if (atomCount > 1) {
       context_.Say(rsrc,
-          "The atomic variable %s should occur exactly once among the arguments of the top-level %s operator"_err_en_US,
+          "The atomic variable %s should be exactly one of the arguments of the top-level %s operator"_err_en_US,
           atom.AsFortran(), operation::ToString(top.first));
+      hasError = true;
     }
-  } else {
+  }
+
+  if (!hasError) {
     CheckStorageOverlap(atom, nonAtom, source);
   }
 }
diff --git a/flang/test/Semantics/OpenMP/atomic-update-only.f90 b/flang/test/Semantics/OpenMP/atomic-update-only.f90
index 28d0e264359c..3c027924a142 100644
--- a/flang/test/Semantics/OpenMP/atomic-update-only.f90
+++ b/flang/test/Semantics/OpenMP/atomic-update-only.f90
@@ -30,7 +30,8 @@ subroutine f03
   integer :: x, y
 
   !$omp atomic update
-  !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+  !ERROR: The atomic variable x cannot be a proper subexpression of an argument (here: (x+y)) in the update operation
+  !ERROR: The atomic variable x should appear as an argument of the top-level + operator
   x = (x + y) + 1
 end
 
diff --git a/flang/test/Semantics/OpenMP/atomic03.f90 b/flang/test/Semantics/OpenMP/atomic03.f90
index b3a3c0d5e7a1..691a483e6e80 100644
--- a/flang/test/Semantics/OpenMP/atomic03.f90
+++ b/flang/test/Semantics/OpenMP/atomic03.f90
@@ -25,19 +25,19 @@ program OmpAtomic
    y = MIN(y, 8)
 
 !$omp atomic
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level AND operator
    z = IAND(y, 4)
 !$omp atomic
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level OR operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level OR operator
    z = IOR(y, 5)
 !$omp atomic
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level NEQV/EOR operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level NEQV/EOR operator
    z = IEOR(y, 6)
 !$omp atomic
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MAX operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level MAX operator
    z = MAX(y, 7, b, c)
 !$omp atomic
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MIN operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level MIN operator
    z = MIN(y, 8, a, d)
 
 !$omp atomic
@@ -58,19 +58,19 @@ program OmpAtomic
    y = MIN(y, 8)
 
 !$omp atomic update
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level AND operator
    z = IAND(y, 4)
 !$omp atomic update 
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level OR operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level OR operator
    z = IOR(y, 5)
 !$omp atomic update
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level NEQV/EOR operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level NEQV/EOR operator
    z = IEOR(y, 6)
 !$omp atomic update
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MAX operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level MAX operator
    z = MAX(y, 7)
 !$omp atomic update
-   !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level MIN operator
+   !ERROR: The atomic variable z should appear as an argument of the top-level MIN operator
    z = MIN(y, 8)
 
 !$omp atomic update
@@ -90,7 +90,7 @@ subroutine conflicting_types()
     type(simple) ::s
     z = 1
     !$omp atomic
-    !ERROR: The atomic variable z should occur exactly once among the arguments of the top-level AND operator
+    !ERROR: The atomic variable z should appear as an argument of the top-level AND operator
     z = IAND(s%z, 4)
 end subroutine
 
@@ -103,22 +103,22 @@ subroutine more_invalid_atomic_update_stmts()
     type(some_type) :: s
  
     !$omp atomic update
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MIN operator
+    !ERROR: The atomic variable a should be exactly one of the arguments of the top-level MIN operator
         a = min(a, a, b)
      
     !$omp atomic
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MAX operator
+    !ERROR: The atomic variable a should be exactly one of the arguments of the top-level MAX operator
         a = max(b, a, b, a)
 
     !$omp atomic
         a = min(b, a, b)
 
     !$omp atomic
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level MAX operator
+    !ERROR: The atomic variable a should be exactly one of the arguments of the top-level MAX operator
         a = max(b, a, b, a, b)
     
     !$omp atomic update
-    !ERROR: The atomic variable y should occur exactly once among the arguments of the top-level MIN operator
+    !ERROR: The atomic variable y should appear as an argument of the top-level MIN operator
         y = min(z, x)
      
     !$omp atomic
@@ -126,7 +126,7 @@ subroutine more_invalid_atomic_update_stmts()
 
     !$omp atomic update
     !ERROR: Atomic variable k should be a scalar
-    !ERROR: The atomic variable k should occur exactly once among the arguments of the top-level MAX operator
+    !ERROR: The atomic variable k should appear as an argument of the top-level MAX operator
         k = max(x, y)
 
     !$omp atomic
diff --git a/flang/test/Semantics/OpenMP/atomic04.f90 b/flang/test/Semantics/OpenMP/atomic04.f90
index 0f69befed141..fb87ca518661 100644
--- a/flang/test/Semantics/OpenMP/atomic04.f90
+++ b/flang/test/Semantics/OpenMP/atomic04.f90
@@ -17,10 +17,10 @@ program OmpAtomic
 !$omp atomic
    x = 1 + x
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level + operator
    x = y + 1
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level + operator
    x = 1 + y
 
 !$omp atomic
@@ -28,10 +28,10 @@ program OmpAtomic
 !$omp atomic
    x = 1 - x
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level - operator
    x = y - 1
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level - operator
    x = 1 - y
 
 !$omp atomic
@@ -50,10 +50,10 @@ program OmpAtomic
 !$omp atomic
    x = 1/x
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level / operator
    x = y/1
 !$omp atomic
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level / operator
    x = 1/y
 
 !$omp atomic
@@ -61,7 +61,7 @@ program OmpAtomic
 !$omp atomic
    m = n .AND. m
 !$omp atomic 
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level AND operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level AND operator
    m = n .AND. l
 
 !$omp atomic
@@ -69,7 +69,7 @@ program OmpAtomic
 !$omp atomic
    m = n .OR. m
 !$omp atomic 
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level OR operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level OR operator
    m = n .OR. l
 
 !$omp atomic
@@ -77,7 +77,7 @@ program OmpAtomic
 !$omp atomic
    m = n .EQV. m
 !$omp atomic
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level EQV operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level EQV operator
    m = n .EQV. l
 
 !$omp atomic
@@ -85,7 +85,7 @@ program OmpAtomic
 !$omp atomic
    m = n .NEQV. m
 !$omp atomic
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level NEQV/EOR operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level NEQV/EOR operator
    m = n .NEQV. l
 
 !$omp atomic update
@@ -93,10 +93,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1 + x
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level + operator
    x = y + 1
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level + operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level + operator
    x = 1 + y
 
 !$omp atomic update
@@ -104,10 +104,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1 - x
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level - operator
    x = y - 1
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level - operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level - operator
    x = 1 - y
 
 !$omp atomic update
@@ -126,10 +126,10 @@ program OmpAtomic
 !$omp atomic update
    x = 1/x
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level / operator
    x = y/1
 !$omp atomic update
-   !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
+   !ERROR: The atomic variable x should appear as an argument of the top-level / operator
    x = 1/y
 
 !$omp atomic update
@@ -137,7 +137,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .AND. m
 !$omp atomic update
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level AND operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level AND operator
    m = n .AND. l
 
 !$omp atomic update
@@ -145,7 +145,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .OR. m
 !$omp atomic update
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level OR operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level OR operator
    m = n .OR. l
 
 !$omp atomic update
@@ -153,7 +153,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .EQV. m
 !$omp atomic update
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level EQV operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level EQV operator
    m = n .EQV. l
 
 !$omp atomic update
@@ -161,7 +161,7 @@ program OmpAtomic
 !$omp atomic update
    m = n .NEQV. m
 !$omp atomic update
-   !ERROR: The atomic variable m should occur exactly once among the arguments of the top-level NEQV/EOR operator
+   !ERROR: The atomic variable m should appear as an argument of the top-level NEQV/EOR operator
    m = n .NEQV. l
 
 end program OmpAtomic
@@ -184,27 +184,30 @@ subroutine more_invalid_atomic_update_stmts()
         x = 1    
 
     !$omp atomic update
-    !ERROR: Within atomic operation a and a*b access the same storage
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: a*b) in the update operation
         a = a * b + a
 
     !$omp atomic
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level * operator
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: (a+9_4)) in the update operation
+    !ERROR: The atomic variable a should appear as an argument of the top-level * operator
         a = b * (a + 9)
 
     !$omp atomic update
-    !ERROR: Within atomic operation a and (a+b) access the same storage
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: (a+b)) in the update operation
         a = a * (a + b)
 
     !$omp atomic
-    !ERROR: Within atomic operation a and (b+a) access the same storage
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: (b+a)) in the update operation
         a = (b + a) * a
 
     !$omp atomic
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level + operator
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: a*b) in the update operation
+    !ERROR: The atomic variable a should appear as an argument of the top-level + operator
         a = a * b + c
 
     !$omp atomic update
-    !ERROR: The atomic variable a should occur exactly once among the arguments of the top-level + operator
+    !ERROR: The atomic variable a cannot be a proper subexpression of an argument (here: a+b) in the update operation
+    !ERROR: The atomic variable a should appear as an argument of the top-level + operator
         a = a + b + c
 
     !$omp atomic
@@ -219,11 +222,12 @@ subroutine more_invalid_atomic_update_stmts()
 
     !$omp atomic update
     !ERROR: No intrinsic or user-defined ASSIGNMENT(=) matches scalar REAL(4) and rank 1 array of REAL(4)
-    !ERROR: The atomic variable x should occur exactly once among the arguments of the top-level / operator
+    !ERROR: The atomic variable x cannot be a proper subexpression of an argument (here: x*y) in the update operation
+    !ERROR: The atomic variable x should appear as an argument of the top-level / operator
         x = x * y / z
 
     !$omp atomic
-    !ERROR: The atomic variable p%m should occur exactly once among the arguments of the top-level + operator
+    !ERROR: The atomic variable p%m should appear as an argument of the top-level + operator
         p%m = x + y
 
     !$omp atomic update
-- 
cgit v1.2.3


From b5aaf9d988ff2dc652c86271b181bf0497eb97cb Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Wed, 18 Jun 2025 08:53:45 -0700
Subject: [InstCombine] Implement vp.reverse reordering/elimination through
 binop/unop (#143963)

This simply copies the structure of the vector.reverse patterns from
just above, and reimplements them for the vp.reverse intrinsics when the
mask is all ones and the EVLs exactly match.

Its unfortunate that we have three different ways to represent a reverse
(shuffle, vector.reverse, and vp.reverse) but I don't see an obvious way
to remove any them because the semantics are slightly different.

This significantly improves vectorization in TSVC_2's s112 and s1112
loops when using EVL tail folding.
---
 .../Transforms/InstCombine/InstCombineCalls.cpp    | 19 ++++++
 .../InstCombine/InstructionCombining.cpp           | 33 +++++++++++
 llvm/test/Transforms/InstCombine/vp-reverse.ll     | 68 ++++++++++++++--------
 3 files changed, 97 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 03897117861f..b6ed1dc4331d 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -3571,6 +3571,25 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
     }
     break;
   }
+  case Intrinsic::experimental_vp_reverse: {
+    Value *X;
+    Value *Vec = II->getArgOperand(0);
+    Value *Mask = II->getArgOperand(1);
+    if (!match(Mask, m_AllOnes()))
+      break;
+    Value *EVL = II->getArgOperand(2);
+    // rev(unop rev(X)) --> unop X
+    if (match(Vec,
+              m_OneUse(m_UnOp(m_Intrinsic<Intrinsic::experimental_vp_reverse>(
+                  m_Value(X), m_AllOnes(), m_Specific(EVL)))))) {
+      auto *OldUnOp = cast<UnaryOperator>(Vec);
+      auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags(
+          OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(),
+          II->getIterator());
+      return replaceInstUsesWith(CI, NewUnOp);
+    }
+    break;
+  }
   case Intrinsic::vector_reduce_or:
   case Intrinsic::vector_reduce_and: {
     // Canonicalize logical or/and reductions:
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index e2cd2a59fab9..afd3359e22ff 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -2231,6 +2231,39 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) {
   else if (isSplatValue(LHS) && match(RHS, m_OneUse(m_VecReverse(m_Value(V2)))))
     return createBinOpReverse(LHS, V2);
 
+  auto createBinOpVPReverse = [&](Value *X, Value *Y, Value *EVL) {
+    Value *V = Builder.CreateBinOp(Opcode, X, Y, Inst.getName());
+    if (auto *BO = dyn_cast<BinaryOperator>(V))
+      BO->copyIRFlags(&Inst);
+
+    ElementCount EC = cast<VectorType>(V->getType())->getElementCount();
+    Value *AllTrueMask = Builder.CreateVectorSplat(EC, Builder.getTrue());
+    Module *M = Inst.getModule();
+    Function *F = Intrinsic::getOrInsertDeclaration(
+        M, Intrinsic::experimental_vp_reverse, V->getType());
+    return CallInst::Create(F, {V, AllTrueMask, EVL});
+  };
+
+  Value *EVL;
+  if (match(LHS, m_Intrinsic<Intrinsic::experimental_vp_reverse>(
+                     m_Value(V1), m_AllOnes(), m_Value(EVL)))) {
+    // Op(rev(V1), rev(V2)) -> rev(Op(V1, V2))
+    if (match(RHS, m_Intrinsic<Intrinsic::experimental_vp_reverse>(
+                       m_Value(V2), m_AllOnes(), m_Specific(EVL))) &&
+        (LHS->hasOneUse() || RHS->hasOneUse() ||
+         (LHS == RHS && LHS->hasNUses(2))))
+      return createBinOpVPReverse(V1, V2, EVL);
+
+    // Op(rev(V1), RHSSplat)) -> rev(Op(V1, RHSSplat))
+    if (LHS->hasOneUse() && isSplatValue(RHS))
+      return createBinOpVPReverse(V1, RHS, EVL);
+  }
+  // Op(LHSSplat, rev(V2)) -> rev(Op(LHSSplat, V2))
+  else if (isSplatValue(LHS) &&
+           match(RHS, m_Intrinsic<Intrinsic::experimental_vp_reverse>(
+                          m_Value(V2), m_AllOnes(), m_Value(EVL))))
+    return createBinOpVPReverse(LHS, V2, EVL);
+
   // It may not be safe to reorder shuffles and things like div, urem, etc.
   // because we may trap when executing those ops on unknown vector elements.
   // See PR20059.
diff --git a/llvm/test/Transforms/InstCombine/vp-reverse.ll b/llvm/test/Transforms/InstCombine/vp-reverse.ll
index 79e6c47bdf1b..540b57da3475 100644
--- a/llvm/test/Transforms/InstCombine/vp-reverse.ll
+++ b/llvm/test/Transforms/InstCombine/vp-reverse.ll
@@ -3,11 +3,8 @@
 
 define <vscale x 4 x i32> @binop_reverse_elim(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i32 %evl) {
 ; CHECK-LABEL: @binop_reverse_elim(
-; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
-; CHECK-NEXT:    [[B:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], [[B]]
-; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD1]]
 ;
   %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   %b.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %b, <vscale x 4 x i1> splat (i1 true), i32 %evl)
@@ -16,8 +13,10 @@ define <vscale x 4 x i32> @binop_reverse_elim(<vscale x 4 x i32> %a, <vscale x 4
   ret <vscale x 4 x i32> %add.rev
 }
 
-define <vscale x 4 x i32> @binop_reverse_elim2(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m, i32 %evl) {
-; CHECK-LABEL: @binop_reverse_elim2(
+; Negative test - the mask needs to be reversed between the inner and
+; the outer to be correct.
+define <vscale x 4 x i32> @binop_reverse_elim_samemask(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %m, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_elim_samemask(
 ; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> [[M:%.*]], i32 [[EVL:%.*]])
 ; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i1> [[M]], i32 [[EVL]])
 ; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV]], [[B_REV]]
@@ -48,10 +47,9 @@ define <vscale x 4 x i32> @binop_reverse_elim_diffmask(<vscale x 4 x i32> %a, <v
 
 define <vscale x 4 x i32> @binop_reverse_elim_diffevl(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, i32 %evl) {
 ; CHECK-LABEL: @binop_reverse_elim_diffevl(
-; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
-; CHECK-NEXT:    [[B_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[B:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV]], [[B_REV]]
-; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]], <vscale x 4 x i1> splat (i1 true), i32 10)
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[A_REV:%.*]], [[B_REV:%.*]]
+; CHECK-NEXT:    [[ADD1:%.*]] = call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
+; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 10)
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
 ;
   %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
@@ -63,10 +61,8 @@ define <vscale x 4 x i32> @binop_reverse_elim_diffevl(<vscale x 4 x i32> %a, <vs
 
 define <vscale x 4 x i32> @binop_reverse_splat_elim(<vscale x 4 x i32> %a, i32 %evl) {
 ; CHECK-LABEL: @binop_reverse_splat_elim(
-; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
-; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], splat (i32 22)
-; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A:%.*]], splat (i32 22)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD1]]
 ;
   %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   %add = add nsw <vscale x 4 x i32> %a.rev, splat (i32 22)
@@ -76,10 +72,8 @@ define <vscale x 4 x i32> @binop_reverse_splat_elim(<vscale x 4 x i32> %a, i32 %
 
 define <vscale x 4 x i32> @binop_reverse_splat_elim2(<vscale x 4 x i32> %a, i32 %evl) {
 ; CHECK-LABEL: @binop_reverse_splat_elim2(
-; CHECK-NEXT:    [[A:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[A1:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
-; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A]], splat (i32 22)
-; CHECK-NEXT:    [[ADD_REV:%.*]] = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse.nxv4i32(<vscale x 4 x i32> [[ADD1]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD_REV]]
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A:%.*]], splat (i32 22)
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD1]]
 ;
   %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   %add = add nsw <vscale x 4 x i32> splat (i32 22), %a.rev
@@ -87,12 +81,40 @@ define <vscale x 4 x i32> @binop_reverse_splat_elim2(<vscale x 4 x i32> %a, i32
   ret <vscale x 4 x i32> %add.rev
 }
 
+define <vscale x 4 x i32> @binop_reverse_splat_elim3(<vscale x 4 x i32> %a, i32 %b, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_splat_elim3(
+; CHECK-NEXT:    [[B_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_VEC:%.*]] = shufflevector <vscale x 4 x i32> [[B_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ADD:%.*]] = add nsw <vscale x 4 x i32> [[B_VEC]], [[A_REV:%.*]]
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD]]
+;
+  %b.ins = insertelement <vscale x 4 x i32> poison, i32 %b, i32 0
+  %b.vec = shufflevector <vscale x 4 x i32> %b.ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %b.vec, %a.rev
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
+define <vscale x 4 x i32> @binop_reverse_splat_elim4(<vscale x 4 x i32> %a, i32 %b, i32 %evl) {
+; CHECK-LABEL: @binop_reverse_splat_elim4(
+; CHECK-NEXT:    [[B_INS:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[B:%.*]], i64 0
+; CHECK-NEXT:    [[B_VEC:%.*]] = shufflevector <vscale x 4 x i32> [[B_INS]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw <vscale x 4 x i32> [[A:%.*]], [[B_VEC]]
+; CHECK-NEXT:    ret <vscale x 4 x i32> [[ADD1]]
+;
+  %b.ins = insertelement <vscale x 4 x i32> poison, i32 %b, i32 0
+  %b.vec = shufflevector <vscale x 4 x i32> %b.ins, <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
+  %a.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  %add = add nsw <vscale x 4 x i32> %a.rev,  %b.vec
+  %add.rev = tail call <vscale x 4 x i32> @llvm.experimental.vp.reverse(<vscale x 4 x i32> %add, <vscale x 4 x i1> splat (i1 true), i32 %evl)
+  ret <vscale x 4 x i32> %add.rev
+}
+
 define <vscale x 4 x float> @unop_reverse_splat_elim(<vscale x 4 x float> %a, <vscale x 4 x float> %b, i32 %evl) {
 ; CHECK-LABEL: @unop_reverse_splat_elim(
-; CHECK-NEXT:    [[A_REV:%.*]] = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[A:%.*]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL:%.*]])
-; CHECK-NEXT:    [[OP:%.*]] = fneg <vscale x 4 x float> [[A_REV]]
-; CHECK-NEXT:    [[OP_REV:%.*]] = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> [[OP]], <vscale x 4 x i1> splat (i1 true), i32 [[EVL]])
-; CHECK-NEXT:    ret <vscale x 4 x float> [[OP_REV]]
+; CHECK-NEXT:    [[OP:%.*]] = fneg <vscale x 4 x float> [[A_REV:%.*]]
+; CHECK-NEXT:    ret <vscale x 4 x float> [[OP]]
 ;
   %a.rev = tail call <vscale x 4 x float> @llvm.experimental.vp.reverse.nxv4f32(<vscale x 4 x float> %a, <vscale x 4 x i1> splat (i1 true), i32 %evl)
   %op = fneg <vscale x 4 x float> %a.rev
-- 
cgit v1.2.3


From 0fa373c77ded203eddb973c79244c75ee5957eaf Mon Sep 17 00:00:00 2001
From: Jon Roelofs <jonathan_roelofs@apple.com>
Date: Wed, 18 Jun 2025 12:00:48 -0400
Subject: [Matrix] Propagate shape information through PHI insts (#141681)

... and split them as we lower them, avoiding several shuffles in the
process.
---
 .../Transforms/Scalar/LowerMatrixIntrinsics.cpp    |  57 +-
 llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll  | 789 +++++++++++++++++++++
 .../propagate-backwards-unsupported.ll             | 261 -------
 3 files changed, 844 insertions(+), 263 deletions(-)
 create mode 100644 llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll

diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 96b156494fd9..fa9e44617b7c 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -288,6 +288,7 @@ static bool isUniformShape(Value *V) {
     }
 
   switch (I->getOpcode()) {
+  case Instruction::PHI:
   case Instruction::FNeg:
     return true;
   default:
@@ -1136,7 +1137,27 @@ public:
 
     Changed |= !FusedInsts.empty();
 
-    // Fourth, lower remaining instructions with shape information.
+    // Fourth, pre-process all the PHINode's. The incoming values will be
+    // assigned later in VisitPHI.
+    for (Instruction *Inst : MatrixInsts) {
+      auto *PHI = dyn_cast<PHINode>(Inst);
+      if (!PHI)
+        continue;
+
+      const ShapeInfo &SI = ShapeMap.at(Inst);
+      auto *EltTy = cast<FixedVectorType>(PHI->getType())->getElementType();
+      MatrixTy PhiM(SI.NumRows, SI.NumColumns, EltTy);
+
+      IRBuilder<> Builder(Inst);
+      for (unsigned VI = 0, VE = PhiM.getNumVectors(); VI != VE; ++VI)
+        PhiM.setVector(VI, Builder.CreatePHI(PhiM.getVectorTy(),
+                                             PHI->getNumIncomingValues(),
+                                             PHI->getName()));
+      assert(!Inst2ColumnMatrix.contains(PHI) && "map already contains phi?");
+      Inst2ColumnMatrix[PHI] = PhiM;
+    }
+
+    // Fifth, lower remaining instructions with shape information.
     for (Instruction *Inst : MatrixInsts) {
       if (FusedInsts.count(Inst))
         continue;
@@ -1161,6 +1182,8 @@ public:
         Result = VisitLoad(cast<LoadInst>(Inst), SI, Op1, Builder);
       else if (match(Inst, m_Store(m_Value(Op1), m_Value(Op2))))
         Result = VisitStore(cast<StoreInst>(Inst), SI, Op1, Op2, Builder);
+      else if (auto *PHI = dyn_cast<PHINode>(Inst))
+        Result = VisitPHI(PHI, SI, Builder);
       else
         continue;
 
@@ -1458,7 +1481,8 @@ public:
                         IRBuilder<> &Builder) {
     auto inserted = Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix));
     (void)inserted;
-    assert(inserted.second && "multiple matrix lowering mapping");
+    assert((inserted.second || isa<PHINode>(Inst)) &&
+           "multiple matrix lowering mapping");
 
     ToRemove.push_back(Inst);
     Value *Flattened = nullptr;
@@ -2239,6 +2263,35 @@ public:
                       Builder);
   }
 
+  MatrixTy VisitPHI(PHINode *Inst, const ShapeInfo &SI, IRBuilder<> &Builder) {
+    auto BlockIP = Inst->getParent()->getFirstInsertionPt();
+    Builder.SetInsertPoint(BlockIP);
+    MatrixTy PhiM = getMatrix(Inst, SI, Builder);
+
+    for (auto [IncomingV, IncomingB] :
+         llvm::zip_equal(Inst->incoming_values(), Inst->blocks())) {
+      // getMatrix() may insert some instructions to help with reshaping. The
+      // safest place for those is at the top of the block after the rest of the
+      // PHI's. Even better, if we can put it in the incoming block.
+      Builder.SetInsertPoint(BlockIP);
+      if (auto *IncomingInst = dyn_cast<Instruction>(IncomingV))
+        if (auto MaybeIP = IncomingInst->getInsertionPointAfterDef())
+          Builder.SetInsertPoint(*MaybeIP);
+
+      MatrixTy OpM = getMatrix(IncomingV, SI, Builder);
+
+      for (unsigned VI = 0, VE = PhiM.getNumVectors(); VI != VE; ++VI) {
+        PHINode *NewPHI = cast<PHINode>(PhiM.getVector(VI));
+        NewPHI->addIncoming(OpM.getVector(VI), IncomingB);
+      }
+    }
+
+    // finalizeLowering() may also insert instructions in some cases. The safe
+    // place for those is at the end of the initial block of PHIs.
+    Builder.SetInsertPoint(BlockIP);
+    return PhiM;
+  }
+
   /// Lower binary operators.
   MatrixTy VisitBinaryOperator(BinaryOperator *Inst, const ShapeInfo &SI,
                                IRBuilder<> &Builder) {
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll
new file mode 100644
index 000000000000..9fdb2ce4dfa7
--- /dev/null
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/phi.ll
@@ -0,0 +1,789 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -matrix-allow-contract=false -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
+
+define void @matrix_phi_loop(ptr %in1, ptr %in2, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN1:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN1]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN1]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI9:%.*]] = phi <3 x double> [ [[COL_LOAD]], [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI10:%.*]] = phi <3 x double> [ [[COL_LOAD1]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI11:%.*]] = phi <3 x double> [ [[COL_LOAD3]], [[ENTRY]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[IN2]], i64 3
+; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[IN2]], i64 6
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[PHI9]], [[COL_LOAD4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[PHI10]], [[COL_LOAD6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[PHI11]], [[COL_LOAD8]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP7]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <3 x double> [[TMP8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> [[TMP4]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP6]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP12:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP7]], ptr [[VEC_GEP12]], align 8
+; CHECK-NEXT:    [[VEC_GEP13:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[TMP8]], ptr [[VEC_GEP13]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in1v = call <9 x double> @llvm.matrix.column.major.load(ptr %in1, i64 3, i1 false, i32 3, i32 3)
+  br label %loop
+
+loop:
+  %phi = phi <9 x double> [%in1v, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %in2v = call <9 x double> @llvm.matrix.column.major.load(ptr %in2, i64 3, i1 false, i32 3, i32 3)
+
+  %sum = fadd <9 x double> %phi, %in2v
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_zeroinitializer(ptr %in1, ptr %in2, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_zeroinitializer(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI4:%.*]] = phi <3 x double> [ zeroinitializer, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI5:%.*]] = phi <3 x double> [ zeroinitializer, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI6:%.*]] = phi <3 x double> [ zeroinitializer, [[ENTRY]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN2]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN2]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[PHI4]], [[COL_LOAD]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[PHI5]], [[COL_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[PHI6]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP7]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <3 x double> [[TMP8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> [[TMP4]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP6]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP7]], ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[TMP8]], ptr [[VEC_GEP8]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi <9 x double> [zeroinitializer, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %inv = call <9 x double> @llvm.matrix.column.major.load(ptr %in2, i64 3, i1 false, i32 3, i32 3)
+
+  %sum = fadd <9 x double> %phi, %inv
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_undef(ptr %in, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_undef(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI:%.*]] = phi <9 x double> [ undef, [[ENTRY:%.*]] ], [ [[SUM:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[INV:%.*]] = load <9 x double>, ptr [[IN:%.*]], align 128
+; CHECK-NEXT:    [[SUM]] = fadd <9 x double> [[PHI]], [[INV]]
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <9 x double> [[SUM]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi <9 x double> [undef, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %inv = load <9 x double>, ptr %in
+
+  %sum = fadd <9 x double> %phi, %inv
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_poison(ptr %in, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_poison(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI4:%.*]] = phi <3 x double> [ poison, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI5:%.*]] = phi <3 x double> [ poison, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI6:%.*]] = phi <3 x double> [ poison, [[ENTRY]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN2]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN2]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[PHI4]], [[COL_LOAD]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[PHI5]], [[COL_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[PHI6]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP7]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <3 x double> [[TMP8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> [[TMP4]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP6]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP7]], ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[TMP8]], ptr [[VEC_GEP8]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi <9 x double> [poison, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %inv = call <9 x double> @llvm.matrix.column.major.load(ptr %in, i64 3, i1 false, i32 3, i32 3)
+
+  %sum = fadd <9 x double> %phi, %inv
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_cdv(ptr %in, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_cdv(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI4:%.*]] = phi <3 x double> [ <double 0.000000e+00, double 1.000000e+00, double 2.000000e+00>, [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI5:%.*]] = phi <3 x double> [ <double 3.000000e+00, double 4.000000e+00, double 5.000000e+00>, [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI6:%.*]] = phi <3 x double> [ <double 6.000000e+00, double 7.000000e+00, double 8.000000e+00>, [[ENTRY]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN2]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN2]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[PHI4]], [[COL_LOAD]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[PHI5]], [[COL_LOAD1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[PHI6]], [[COL_LOAD3]]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP7]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <3 x double> [[TMP8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <6 x double> [[TMP3]], <6 x double> [[TMP4]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP1]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP2]] = shufflevector <9 x double> [[TMP5]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP6]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP7]], ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    [[VEC_GEP8:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[TMP8]], ptr [[VEC_GEP8]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %phi = phi <9 x double> [<double 0., double 1., double 2., double 3., double 4., double 5., double 6., double 7., double 8.>, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %inv = call <9 x double> @llvm.matrix.column.major.load(ptr %in, i64 3, i1 false, i32 3, i32 3)
+
+  %sum = fadd <9 x double> %phi, %inv
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_delay(ptr %in1, ptr %in2, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_delay(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN1:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN1]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN1]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI14:%.*]] = phi <3 x double> [ [[COL_LOAD]], [[ENTRY:%.*]] ], [ [[TMP0:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI15:%.*]] = phi <3 x double> [ [[COL_LOAD1]], [[ENTRY]] ], [ [[TMP1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI16:%.*]] = phi <3 x double> [ [[COL_LOAD3]], [[ENTRY]] ], [ [[TMP2:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0]] = phi <3 x double> [ [[COL_LOAD]], [[ENTRY]] ], [ [[SPLIT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP1]] = phi <3 x double> [ [[COL_LOAD1]], [[ENTRY]] ], [ [[SPLIT10:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP2]] = phi <3 x double> [ [[COL_LOAD3]], [[ENTRY]] ], [ [[SPLIT11:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[IN2]], i64 3
+; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[IN2]], i64 6
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[PHI14]], [[COL_LOAD4]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[PHI15]], [[COL_LOAD6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[PHI16]], [[COL_LOAD8]]
+; CHECK-NEXT:    [[TMP9:%.*]] = shufflevector <3 x double> [[TMP6]], <3 x double> [[TMP7]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP10:%.*]] = shufflevector <3 x double> [[TMP8]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP11:%.*]] = shufflevector <6 x double> [[TMP9]], <6 x double> [[TMP10]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT]] = shufflevector <9 x double> [[TMP11]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT10]] = shufflevector <9 x double> [[TMP11]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT11]] = shufflevector <9 x double> [[TMP11]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP12:%.*]] = fadd <3 x double> [[TMP6]], [[TMP0]]
+; CHECK-NEXT:    [[TMP13:%.*]] = fadd <3 x double> [[TMP7]], [[TMP1]]
+; CHECK-NEXT:    [[TMP14:%.*]] = fadd <3 x double> [[TMP8]], [[TMP2]]
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP12]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP12:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP13]], ptr [[VEC_GEP12]], align 8
+; CHECK-NEXT:    [[VEC_GEP13:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[TMP14]], ptr [[VEC_GEP13]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in1v = call <9 x double> @llvm.matrix.column.major.load(ptr %in1, i64 3, i1 false, i32 3, i32 3)
+  br label %loop
+
+loop:
+  %phi2 = phi <9 x double> [%in1v, %entry], [%phi, %loop]
+  %phi = phi <9 x double> [%in1v, %entry], [%sum, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %in2v = call <9 x double> @llvm.matrix.column.major.load(ptr %in2, i64 3, i1 false, i32 3, i32 3)
+
+  %sum = fadd <9 x double> %phi2, %in2v
+  %sum2 = fadd <9 x double> %sum, %phi
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <9 x double> %sum2, ptr %out
+  ret void
+}
+
+define void @matrix_phi_loop_delay_reshape(ptr %in1, ptr %in2, ptr %in3, i32 %count, ptr %out) {
+; CHECK-LABEL: @matrix_phi_loop_delay_reshape(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[IN3:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[IN3]], i64 2
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <2 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    [[VEC_GEP1:%.*]] = getelementptr double, ptr [[IN3]], i64 4
+; CHECK-NEXT:    [[COL_LOAD12:%.*]] = load <2 x double>, ptr [[VEC_GEP1]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> [[COL_LOAD8]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x double> [[COL_LOAD12]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> [[TMP1]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <6 x double> [[TMP2]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[COL_LOAD10:%.*]] = shufflevector <6 x double> [[TMP2]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <6 x double>, ptr [[IN2:%.*]], align 8
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[IN1:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[IN1]], i64 3
+; CHECK-NEXT:    [[COL_LOAD14:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[PHI2:%.*]] = phi <3 x double> [ [[SPLIT]], [[ENTRY:%.*]] ], [ [[PHI1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI39:%.*]] = phi <3 x double> [ [[COL_LOAD10]], [[ENTRY]] ], [ [[PHI4:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI25:%.*]] = phi <6 x double> [ [[COL_LOAD11]], [[ENTRY]] ], [ [[PHI25]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI1]] = phi <3 x double> [ [[COL_LOAD]], [[ENTRY]] ], [ [[PHI2]], [[LOOP]] ]
+; CHECK-NEXT:    [[PHI4]] = phi <3 x double> [ [[COL_LOAD14]], [[ENTRY]] ], [ [[PHI39]], [[LOOP]] ]
+; CHECK-NEXT:    [[CTR:%.*]] = phi i32 [ [[COUNT:%.*]], [[ENTRY]] ], [ [[DEC:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP3:%.*]] = shufflevector <6 x double> [[PHI25]], <6 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <6 x double> [[PHI25]], <6 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP7:%.*]] = fadd <3 x double> [[TMP3]], [[PHI1]]
+; CHECK-NEXT:    [[TMP8:%.*]] = fadd <3 x double> [[TMP4]], [[PHI4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = fadd <3 x double> [[TMP7]], [[PHI2]]
+; CHECK-NEXT:    [[TMP6:%.*]] = fadd <3 x double> [[TMP8]], [[PHI39]]
+; CHECK-NEXT:    [[DEC]] = sub i32 [[CTR]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[DEC]], 0
+; CHECK-NEXT:    br i1 [[CMP]], label [[EXIT:%.*]], label [[LOOP]]
+; CHECK:       exit:
+; CHECK-NEXT:    store <3 x double> [[TMP5]], ptr [[OUT:%.*]], align 64
+; CHECK-NEXT:    [[VEC_GEP30:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[TMP6]], ptr [[VEC_GEP30]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  %in1v = call <6 x double> @llvm.matrix.column.major.load(ptr %in3, i64 2, i1 false, i32 2, i32 3)
+  %in2v = call <6 x double> @llvm.matrix.column.major.load(ptr %in2, i64 6, i1 false, i32 6, i32 1)
+  %in3v = call <6 x double> @llvm.matrix.column.major.load(ptr %in1, i64 3, i1 false, i32 3, i32 2)
+  br label %loop
+
+loop:
+  %phi = phi <6 x double> [%in1v, %entry], [%phi3, %loop]
+  %phi2 = phi <6 x double> [%in2v, %entry], [%phi2, %loop]
+  %phi3 = phi <6 x double> [%in3v, %entry], [%phi, %loop]
+  %ctr = phi i32 [%count, %entry], [%dec, %loop]
+
+  %sum = fadd <6 x double> %phi2, %phi3
+  %sum2 = fadd <6 x double> %sum, %phi
+
+  %dec = sub i32 %ctr, 1
+  %cmp = icmp eq i32 %dec, 0
+  br i1 %cmp, label %exit, label %loop
+
+exit:
+  store <6 x double> %sum2, ptr %out
+  ret void
+}
+
+define void @matrix_phi_three_preds(i1 %cond1, i1 %cond2, ptr %a, ptr %b, ptr %c, ptr %out) {
+; CHECK-LABEL: @matrix_phi_three_preds(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[BB1:%.*]], label [[BBA:%.*]]
+; CHECK:       bb1:
+; CHECK-NEXT:    br i1 [[COND2:%.*]], label [[BBB:%.*]], label [[BBC:%.*]]
+; CHECK:       bba:
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       bbb:
+; CHECK-NEXT:    [[COL_LOAD9:%.*]] = load <3 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr double, ptr [[B]], i64 3
+; CHECK-NEXT:    [[COL_LOAD11:%.*]] = load <3 x double>, ptr [[VEC_GEP10]], align 8
+; CHECK-NEXT:    [[VEC_GEP12:%.*]] = getelementptr double, ptr [[B]], i64 6
+; CHECK-NEXT:    [[COL_LOAD13:%.*]] = load <3 x double>, ptr [[VEC_GEP12]], align 8
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       bbc:
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[C:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP5:%.*]] = getelementptr double, ptr [[C]], i64 3
+; CHECK-NEXT:    [[COL_LOAD6:%.*]] = load <3 x double>, ptr [[VEC_GEP5]], align 8
+; CHECK-NEXT:    [[VEC_GEP7:%.*]] = getelementptr double, ptr [[C]], i64 6
+; CHECK-NEXT:    [[COL_LOAD8:%.*]] = load <3 x double>, ptr [[VEC_GEP7]], align 8
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI14:%.*]] = phi <3 x double> [ [[COL_LOAD]], [[BBA]] ], [ [[COL_LOAD9]], [[BBB]] ], [ [[COL_LOAD4]], [[BBC]] ]
+; CHECK-NEXT:    [[PHI15:%.*]] = phi <3 x double> [ [[COL_LOAD1]], [[BBA]] ], [ [[COL_LOAD11]], [[BBB]] ], [ [[COL_LOAD6]], [[BBC]] ]
+; CHECK-NEXT:    [[PHI16:%.*]] = phi <3 x double> [ [[COL_LOAD3]], [[BBA]] ], [ [[COL_LOAD13]], [[BBB]] ], [ [[COL_LOAD8]], [[BBC]] ]
+; CHECK-NEXT:    store <3 x double> [[PHI14]], ptr [[OUT:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP17:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[PHI15]], ptr [[VEC_GEP17]], align 8
+; CHECK-NEXT:    [[VEC_GEP18:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[PHI16]], ptr [[VEC_GEP18]], align 8
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond1, label %bb1, label %bba
+
+bb1:
+  br i1 %cond2, label %bbb, label %bbc
+
+bba:
+  %va = call <9 x double> @llvm.matrix.column.major.load(ptr %a, i64 3, i1 false, i32 3, i32 3)
+  br label %exit
+
+bbb:
+  %vb = call <9 x double> @llvm.matrix.column.major.load(ptr %b, i64 3, i1 false, i32 3, i32 3)
+  br label %exit
+
+bbc:
+  %vc = call <9 x double> @llvm.matrix.column.major.load(ptr %c, i64 3, i1 false, i32 3, i32 3)
+  br label %exit
+
+exit:
+  %phi = phi <9 x double> [%va, %bba], [%vb, %bbb], [%vc, %bbc]
+  call void @llvm.matrix.column.major.store(<9 x double> %phi, ptr %out, i64 3, i1 false, i32 3, i32 3)
+  ret void
+}
+
+define void @matrix_phi_two_preds_shape_mismatch1(i1 %cond1, ptr %a, ptr %b, ptr %out) {
+; CHECK-LABEL: @matrix_phi_two_preds_shape_mismatch1(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[BBA:%.*]], label [[BBB:%.*]]
+; CHECK:       bba:
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[A]], i64 3
+; CHECK-NEXT:    [[COL_LOAD2:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP3:%.*]] = getelementptr double, ptr [[A]], i64 6
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <3 x double>, ptr [[VEC_GEP3]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = shufflevector <3 x double> [[COL_LOAD1]], <3 x double> [[COL_LOAD2]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <3 x double> [[COL_LOAD4]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <6 x double> [[TMP0]], <6 x double> [[TMP1]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[TMP2]], <9 x double> poison, <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       bbb:
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <9 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI5:%.*]] = phi <9 x double> [ [[SPLIT]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
+; CHECK-NEXT:    store <9 x double> [[PHI5]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond1, label %bba, label %bbb
+
+bba:
+  %va = call <9 x double> @llvm.matrix.column.major.load(ptr %a, i64 3, i1 false, i32 3, i32 3)
+  br label %exit
+
+bbb:
+  %vb = call <9 x double> @llvm.matrix.column.major.load(ptr %b, i64 9, i1 false, i32 9, i32 1)
+  br label %exit
+
+exit:
+  %phi = phi <9 x double> [%va, %bba], [%vb, %bbb]
+  store <9 x double> %phi, ptr %out
+  ret void
+}
+
+define void @matrix_phi_two_preds_shape_mismatch2(i1 %cond1, ptr %a, ptr %b, ptr %out) {
+; CHECK-LABEL: @matrix_phi_two_preds_shape_mismatch2(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND1:%.*]], label [[BBA:%.*]], label [[BBB:%.*]]
+; CHECK:       bba:
+; CHECK-NEXT:    [[COL_LOAD4:%.*]] = load <9 x double>, ptr [[A:%.*]], align 8
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT8:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT9:%.*]] = shufflevector <9 x double> [[COL_LOAD4]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    br label [[EXIT:%.*]]
+; CHECK:       bbb:
+; CHECK-NEXT:    [[COL_LOAD:%.*]] = load <3 x double>, ptr [[B:%.*]], align 8
+; CHECK-NEXT:    [[VEC_GEP:%.*]] = getelementptr double, ptr [[B]], i64 3
+; CHECK-NEXT:    [[COL_LOAD1:%.*]] = load <3 x double>, ptr [[VEC_GEP]], align 8
+; CHECK-NEXT:    [[VEC_GEP2:%.*]] = getelementptr double, ptr [[B]], i64 6
+; CHECK-NEXT:    [[COL_LOAD3:%.*]] = load <3 x double>, ptr [[VEC_GEP2]], align 8
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[PHI5:%.*]] = phi <3 x double> [ [[SPLIT]], [[BBA]] ], [ [[COL_LOAD]], [[BBB]] ]
+; CHECK-NEXT:    [[PHI6:%.*]] = phi <3 x double> [ [[SPLIT8]], [[BBA]] ], [ [[COL_LOAD1]], [[BBB]] ]
+; CHECK-NEXT:    [[PHI7:%.*]] = phi <3 x double> [ [[SPLIT9]], [[BBA]] ], [ [[COL_LOAD3]], [[BBB]] ]
+; CHECK-NEXT:    store <3 x double> [[PHI5]], ptr [[OUT:%.*]], align 128
+; CHECK-NEXT:    [[VEC_GEP10:%.*]] = getelementptr double, ptr [[OUT]], i64 3
+; CHECK-NEXT:    store <3 x double> [[PHI6]], ptr [[VEC_GEP10]], align 8
+; CHECK-NEXT:    [[VEC_GEP11:%.*]] = getelementptr double, ptr [[OUT]], i64 6
+; CHECK-NEXT:    store <3 x double> [[PHI7]], ptr [[VEC_GEP11]], align 16
+; CHECK-NEXT:    ret void
+;
+entry:
+  br i1 %cond1, label %bba, label %bbb
+
+bba:
+  %va = call <9 x double> @llvm.matrix.column.major.load(ptr %a, i64 9, i1 false, i32 9, i32 1)
+  br label %exit
+
+bbb:
+  %vb = call <9 x double> @llvm.matrix.column.major.load(ptr %b, i64 3, i1 false, i32 3, i32 3)
+  br label %exit
+
+exit:
+  %phi = phi <9 x double> [%va, %bba], [%vb, %bbb]
+  store <9 x double> %phi, ptr %out
+  ret void
+}
+
+define <9 x double> @matrix_phi_ifthenelse(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) {
+; CHECK-LABEL: @matrix_phi_ifthenelse(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+; CHECK:       if.then:
+; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0
+; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1
+; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1
+; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2
+; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2
+; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2
+; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
+; CHECK-NEXT:    br label [[IF_END:%.*]]
+; CHECK:       if.else:
+; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <9 x double> [[B]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[B]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
+; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <3 x double> poison, double [[TMP21]], i64 0
+; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <3 x double> [[TMP22]], double [[TMP23]], i64 1
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <3 x double> [[TMP24]], double [[TMP25]], i64 2
+; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <3 x double> poison, double [[TMP27]], i64 0
+; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <3 x double> [[TMP28]], double [[TMP29]], i64 1
+; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <3 x double> [[TMP30]], double [[TMP31]], i64 2
+; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2
+; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <3 x double> poison, double [[TMP33]], i64 0
+; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2
+; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <3 x double> [[TMP34]], double [[TMP35]], i64 1
+; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
+; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2
+; CHECK-NEXT:    br label [[IF_END]]
+; CHECK:       if.end:
+; CHECK-NEXT:    [[MERGE9:%.*]] = phi <3 x double> [ [[TMP5]], [[IF_THEN]] ], [ [[TMP26]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[MERGE10:%.*]] = phi <3 x double> [ [[TMP11]], [[IF_THEN]] ], [ [[TMP32]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[MERGE11:%.*]] = phi <3 x double> [ [[TMP17]], [[IF_THEN]] ], [ [[TMP38]], [[IF_ELSE]] ]
+; CHECK-NEXT:    [[SPLIT9:%.*]] = shufflevector <9 x double> [[MERGE:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT:    [[SPLIT10:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[SPLIT11:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
+; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[MERGE9]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
+; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <3 x double> [[MERGE9]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP44]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
+; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
+; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <3 x double> [[MERGE9]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP47]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP48:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
+; CHECK-NEXT:    [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
+; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP50]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[MERGE9]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP53:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
+; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <3 x double> [[MERGE9]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP54]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP55:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
+; CHECK-NEXT:    [[TMP56:%.*]] = fadd <1 x double> [[TMP53]], [[TMP55]]
+; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP57:%.*]] = extractelement <3 x double> [[MERGE9]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP57]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP58:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
+; CHECK-NEXT:    [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
+; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[MERGE9]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP63:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
+; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <3 x double> [[MERGE9]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP64]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP65:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
+; CHECK-NEXT:    [[TMP66:%.*]] = fadd <1 x double> [[TMP63]], [[TMP65]]
+; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <3 x double> [[MERGE9]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP67]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP68:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
+; CHECK-NEXT:    [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]]
+; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[MERGE10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP73:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
+; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <3 x double> [[MERGE10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP74]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP75:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
+; CHECK-NEXT:    [[TMP76:%.*]] = fadd <1 x double> [[TMP73]], [[TMP75]]
+; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <3 x double> [[MERGE10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP77]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP78:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
+; CHECK-NEXT:    [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]]
+; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP80]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[MERGE10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP83:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
+; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <3 x double> [[MERGE10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP84]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP85:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
+; CHECK-NEXT:    [[TMP86:%.*]] = fadd <1 x double> [[TMP83]], [[TMP85]]
+; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <3 x double> [[MERGE10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP87]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP88:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
+; CHECK-NEXT:    [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]]
+; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP91:%.*]] = shufflevector <3 x double> [[TMP81]], <3 x double> [[TMP90]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[MERGE10]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP93:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
+; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <3 x double> [[MERGE10]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP94]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP95:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
+; CHECK-NEXT:    [[TMP96:%.*]] = fadd <1 x double> [[TMP93]], [[TMP95]]
+; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <3 x double> [[MERGE10]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP97]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP98:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
+; CHECK-NEXT:    [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]]
+; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[MERGE11]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP103:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
+; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <3 x double> [[MERGE11]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP104]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP105:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
+; CHECK-NEXT:    [[TMP106:%.*]] = fadd <1 x double> [[TMP103]], [[TMP105]]
+; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP107:%.*]] = extractelement <3 x double> [[MERGE11]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP107]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP108:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
+; CHECK-NEXT:    [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]]
+; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP111:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP110]], <3 x i32> <i32 3, i32 1, i32 2>
+; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP112:%.*]] = extractelement <3 x double> [[MERGE11]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP112]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP113:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
+; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <3 x double> [[MERGE11]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP114]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP115:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
+; CHECK-NEXT:    [[TMP116:%.*]] = fadd <1 x double> [[TMP113]], [[TMP115]]
+; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 1>
+; CHECK-NEXT:    [[TMP117:%.*]] = extractelement <3 x double> [[MERGE11]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP117]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP118:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
+; CHECK-NEXT:    [[TMP119:%.*]] = fadd <1 x double> [[TMP116]], [[TMP118]]
+; CHECK-NEXT:    [[TMP120:%.*]] = shufflevector <1 x double> [[TMP119]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP121:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> [[TMP120]], <3 x i32> <i32 0, i32 3, i32 2>
+; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[SPLIT9]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <3 x double> [[MERGE11]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP122]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP123:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
+; CHECK-NEXT:    [[BLOCK84:%.*]] = shufflevector <3 x double> [[SPLIT10]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <3 x double> [[MERGE11]], i64 1
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP124]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP125:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
+; CHECK-NEXT:    [[TMP126:%.*]] = fadd <1 x double> [[TMP123]], [[TMP125]]
+; CHECK-NEXT:    [[BLOCK87:%.*]] = shufflevector <3 x double> [[SPLIT11]], <3 x double> poison, <1 x i32> <i32 2>
+; CHECK-NEXT:    [[TMP127:%.*]] = extractelement <3 x double> [[MERGE11]], i64 2
+; CHECK-NEXT:    [[SPLAT_SPLATINSERT88:%.*]] = insertelement <1 x double> poison, double [[TMP127]], i64 0
+; CHECK-NEXT:    [[SPLAT_SPLAT89:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT88]], <1 x double> poison, <1 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP128:%.*]] = fmul <1 x double> [[BLOCK87]], [[SPLAT_SPLAT89]]
+; CHECK-NEXT:    [[TMP129:%.*]] = fadd <1 x double> [[TMP126]], [[TMP128]]
+; CHECK-NEXT:    [[TMP130:%.*]] = shufflevector <1 x double> [[TMP129]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP131:%.*]] = shufflevector <3 x double> [[TMP121]], <3 x double> [[TMP130]], <3 x i32> <i32 0, i32 1, i32 3>
+; CHECK-NEXT:    [[TMP132:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP101]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
+; CHECK-NEXT:    [[TMP133:%.*]] = shufflevector <3 x double> [[TMP131]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
+; CHECK-NEXT:    [[TMP134:%.*]] = shufflevector <6 x double> [[TMP132]], <6 x double> [[TMP133]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
+; CHECK-NEXT:    ret <9 x double> [[TMP134]]
+;
+entry:
+  br i1 %cond, label %if.then, label %if.else
+
+if.then:                    ; preds = %entry
+  %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3)
+  br label %if.end
+
+if.else:                                       ; preds = %entry
+  %B.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %B, i32 3, i32 3)
+  br label %if.end
+
+if.end:                                        ; preds = %if.then, %if.else
+  %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ]
+  %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3)
+  ret <9 x double> %res
+}
diff --git a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
index 2af2c979f206..f07e1762d404 100644
--- a/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
+++ b/llvm/test/Transforms/LowerMatrixIntrinsics/propagate-backwards-unsupported.ll
@@ -1,267 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -passes='lower-matrix-intrinsics' -S < %s | FileCheck %s
 
-; Check that we we use flattened vectors for PHI operands and extract the columns afterwards.
-define <9 x double> @unsupported_phi(i1 %cond, <9 x double> %A, <9 x double> %B, <9 x double> %C) {
-; CHECK-LABEL: @unsupported_phi(
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 [[COND:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
-; CHECK:       if.then:
-; CHECK-NEXT:    [[SPLIT3:%.*]] = shufflevector <9 x double> [[A:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[SPLIT4:%.*]] = shufflevector <9 x double> [[A]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[SPLIT5:%.*]] = shufflevector <9 x double> [[A]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
-; CHECK-NEXT:    [[TMP0:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = insertelement <3 x double> poison, double [[TMP0]], i64 0
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 0
-; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x double> [[TMP1]], double [[TMP2]], i64 1
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 0
-; CHECK-NEXT:    [[TMP5:%.*]] = insertelement <3 x double> [[TMP3]], double [[TMP4]], i64 2
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 1
-; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <3 x double> poison, double [[TMP6]], i64 0
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 1
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <3 x double> [[TMP7]], double [[TMP8]], i64 1
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 1
-; CHECK-NEXT:    [[TMP11:%.*]] = insertelement <3 x double> [[TMP9]], double [[TMP10]], i64 2
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <3 x double> [[SPLIT3]], i64 2
-; CHECK-NEXT:    [[TMP13:%.*]] = insertelement <3 x double> poison, double [[TMP12]], i64 0
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <3 x double> [[SPLIT4]], i64 2
-; CHECK-NEXT:    [[TMP15:%.*]] = insertelement <3 x double> [[TMP13]], double [[TMP14]], i64 1
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <3 x double> [[SPLIT5]], i64 2
-; CHECK-NEXT:    [[TMP17:%.*]] = insertelement <3 x double> [[TMP15]], double [[TMP16]], i64 2
-; CHECK-NEXT:    [[TMP18:%.*]] = shufflevector <3 x double> [[TMP5]], <3 x double> [[TMP11]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP19:%.*]] = shufflevector <3 x double> [[TMP17]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP20:%.*]] = shufflevector <6 x double> [[TMP18]], <6 x double> [[TMP19]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT:    br label [[IF_END:%.*]]
-; CHECK:       if.else:
-; CHECK-NEXT:    [[SPLIT:%.*]] = shufflevector <9 x double> [[B:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[SPLIT1:%.*]] = shufflevector <9 x double> [[B]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[SPLIT2:%.*]] = shufflevector <9 x double> [[B]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <3 x double> [[SPLIT]], i64 0
-; CHECK-NEXT:    [[TMP22:%.*]] = insertelement <3 x double> poison, double [[TMP21]], i64 0
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 0
-; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <3 x double> [[TMP22]], double [[TMP23]], i64 1
-; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 0
-; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <3 x double> [[TMP24]], double [[TMP25]], i64 2
-; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <3 x double> [[SPLIT]], i64 1
-; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <3 x double> poison, double [[TMP27]], i64 0
-; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 1
-; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <3 x double> [[TMP28]], double [[TMP29]], i64 1
-; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 1
-; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <3 x double> [[TMP30]], double [[TMP31]], i64 2
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <3 x double> [[SPLIT]], i64 2
-; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <3 x double> poison, double [[TMP33]], i64 0
-; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <3 x double> [[SPLIT1]], i64 2
-; CHECK-NEXT:    [[TMP36:%.*]] = insertelement <3 x double> [[TMP34]], double [[TMP35]], i64 1
-; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <3 x double> [[SPLIT2]], i64 2
-; CHECK-NEXT:    [[TMP38:%.*]] = insertelement <3 x double> [[TMP36]], double [[TMP37]], i64 2
-; CHECK-NEXT:    [[TMP39:%.*]] = shufflevector <3 x double> [[TMP26]], <3 x double> [[TMP32]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP40:%.*]] = shufflevector <3 x double> [[TMP38]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP41:%.*]] = shufflevector <6 x double> [[TMP39]], <6 x double> [[TMP40]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT:    br label [[IF_END]]
-; CHECK:       if.end:
-; CHECK-NEXT:    [[MERGE:%.*]] = phi <9 x double> [ [[TMP20]], [[IF_THEN]] ], [ [[TMP41]], [[IF_ELSE]] ]
-; CHECK-NEXT:    [[SPLIT6:%.*]] = shufflevector <9 x double> [[C:%.*]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[SPLIT7:%.*]] = shufflevector <9 x double> [[C]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[SPLIT8:%.*]] = shufflevector <9 x double> [[C]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
-; CHECK-NEXT:    [[SPLIT9:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT:    [[SPLIT10:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[SPLIT11:%.*]] = shufflevector <9 x double> [[MERGE]], <9 x double> poison, <3 x i32> <i32 6, i32 7, i32 8>
-; CHECK-NEXT:    [[BLOCK:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP42:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <1 x double> poison, double [[TMP42]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP43:%.*]] = fmul <1 x double> [[BLOCK]], [[SPLAT_SPLAT]]
-; CHECK-NEXT:    [[BLOCK12:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP44:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT13:%.*]] = insertelement <1 x double> poison, double [[TMP44]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT14:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT13]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP45:%.*]] = fmul <1 x double> [[BLOCK12]], [[SPLAT_SPLAT14]]
-; CHECK-NEXT:    [[TMP46:%.*]] = fadd <1 x double> [[TMP43]], [[TMP45]]
-; CHECK-NEXT:    [[BLOCK15:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP47:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT16:%.*]] = insertelement <1 x double> poison, double [[TMP47]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT17:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT16]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP48:%.*]] = fmul <1 x double> [[BLOCK15]], [[SPLAT_SPLAT17]]
-; CHECK-NEXT:    [[TMP49:%.*]] = fadd <1 x double> [[TMP46]], [[TMP48]]
-; CHECK-NEXT:    [[TMP50:%.*]] = shufflevector <1 x double> [[TMP49]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP51:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP50]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT:    [[BLOCK18:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP52:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT19:%.*]] = insertelement <1 x double> poison, double [[TMP52]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT20:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT19]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP53:%.*]] = fmul <1 x double> [[BLOCK18]], [[SPLAT_SPLAT20]]
-; CHECK-NEXT:    [[BLOCK21:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP54:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT22:%.*]] = insertelement <1 x double> poison, double [[TMP54]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT23:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT22]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP55:%.*]] = fmul <1 x double> [[BLOCK21]], [[SPLAT_SPLAT23]]
-; CHECK-NEXT:    [[TMP56:%.*]] = fadd <1 x double> [[TMP53]], [[TMP55]]
-; CHECK-NEXT:    [[BLOCK24:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP57:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT25:%.*]] = insertelement <1 x double> poison, double [[TMP57]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT26:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT25]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP58:%.*]] = fmul <1 x double> [[BLOCK24]], [[SPLAT_SPLAT26]]
-; CHECK-NEXT:    [[TMP59:%.*]] = fadd <1 x double> [[TMP56]], [[TMP58]]
-; CHECK-NEXT:    [[TMP60:%.*]] = shufflevector <1 x double> [[TMP59]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP61:%.*]] = shufflevector <3 x double> [[TMP51]], <3 x double> [[TMP60]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[BLOCK27:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP62:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT28:%.*]] = insertelement <1 x double> poison, double [[TMP62]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT29:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT28]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP63:%.*]] = fmul <1 x double> [[BLOCK27]], [[SPLAT_SPLAT29]]
-; CHECK-NEXT:    [[BLOCK30:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP64:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT31:%.*]] = insertelement <1 x double> poison, double [[TMP64]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT32:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT31]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP65:%.*]] = fmul <1 x double> [[BLOCK30]], [[SPLAT_SPLAT32]]
-; CHECK-NEXT:    [[TMP66:%.*]] = fadd <1 x double> [[TMP63]], [[TMP65]]
-; CHECK-NEXT:    [[BLOCK33:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP67:%.*]] = extractelement <3 x double> [[SPLIT9]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT34:%.*]] = insertelement <1 x double> poison, double [[TMP67]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT35:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT34]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP68:%.*]] = fmul <1 x double> [[BLOCK33]], [[SPLAT_SPLAT35]]
-; CHECK-NEXT:    [[TMP69:%.*]] = fadd <1 x double> [[TMP66]], [[TMP68]]
-; CHECK-NEXT:    [[TMP70:%.*]] = shufflevector <1 x double> [[TMP69]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP71:%.*]] = shufflevector <3 x double> [[TMP61]], <3 x double> [[TMP70]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT:    [[BLOCK36:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP72:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT37:%.*]] = insertelement <1 x double> poison, double [[TMP72]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT38:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT37]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP73:%.*]] = fmul <1 x double> [[BLOCK36]], [[SPLAT_SPLAT38]]
-; CHECK-NEXT:    [[BLOCK39:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP74:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT40:%.*]] = insertelement <1 x double> poison, double [[TMP74]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT41:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT40]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP75:%.*]] = fmul <1 x double> [[BLOCK39]], [[SPLAT_SPLAT41]]
-; CHECK-NEXT:    [[TMP76:%.*]] = fadd <1 x double> [[TMP73]], [[TMP75]]
-; CHECK-NEXT:    [[BLOCK42:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP77:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT43:%.*]] = insertelement <1 x double> poison, double [[TMP77]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT44:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT43]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP78:%.*]] = fmul <1 x double> [[BLOCK42]], [[SPLAT_SPLAT44]]
-; CHECK-NEXT:    [[TMP79:%.*]] = fadd <1 x double> [[TMP76]], [[TMP78]]
-; CHECK-NEXT:    [[TMP80:%.*]] = shufflevector <1 x double> [[TMP79]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP81:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP80]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT:    [[BLOCK45:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP82:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT46:%.*]] = insertelement <1 x double> poison, double [[TMP82]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT47:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT46]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP83:%.*]] = fmul <1 x double> [[BLOCK45]], [[SPLAT_SPLAT47]]
-; CHECK-NEXT:    [[BLOCK48:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP84:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT49:%.*]] = insertelement <1 x double> poison, double [[TMP84]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT50:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT49]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP85:%.*]] = fmul <1 x double> [[BLOCK48]], [[SPLAT_SPLAT50]]
-; CHECK-NEXT:    [[TMP86:%.*]] = fadd <1 x double> [[TMP83]], [[TMP85]]
-; CHECK-NEXT:    [[BLOCK51:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP87:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT52:%.*]] = insertelement <1 x double> poison, double [[TMP87]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT53:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT52]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP88:%.*]] = fmul <1 x double> [[BLOCK51]], [[SPLAT_SPLAT53]]
-; CHECK-NEXT:    [[TMP89:%.*]] = fadd <1 x double> [[TMP86]], [[TMP88]]
-; CHECK-NEXT:    [[TMP90:%.*]] = shufflevector <1 x double> [[TMP89]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP91:%.*]] = shufflevector <3 x double> [[TMP81]], <3 x double> [[TMP90]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[BLOCK54:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP92:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT55:%.*]] = insertelement <1 x double> poison, double [[TMP92]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT56:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT55]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP93:%.*]] = fmul <1 x double> [[BLOCK54]], [[SPLAT_SPLAT56]]
-; CHECK-NEXT:    [[BLOCK57:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP94:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT58:%.*]] = insertelement <1 x double> poison, double [[TMP94]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT59:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT58]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP95:%.*]] = fmul <1 x double> [[BLOCK57]], [[SPLAT_SPLAT59]]
-; CHECK-NEXT:    [[TMP96:%.*]] = fadd <1 x double> [[TMP93]], [[TMP95]]
-; CHECK-NEXT:    [[BLOCK60:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP97:%.*]] = extractelement <3 x double> [[SPLIT10]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT61:%.*]] = insertelement <1 x double> poison, double [[TMP97]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT62:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT61]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP98:%.*]] = fmul <1 x double> [[BLOCK60]], [[SPLAT_SPLAT62]]
-; CHECK-NEXT:    [[TMP99:%.*]] = fadd <1 x double> [[TMP96]], [[TMP98]]
-; CHECK-NEXT:    [[TMP100:%.*]] = shufflevector <1 x double> [[TMP99]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP101:%.*]] = shufflevector <3 x double> [[TMP91]], <3 x double> [[TMP100]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT:    [[BLOCK63:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP102:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT64:%.*]] = insertelement <1 x double> poison, double [[TMP102]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT65:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT64]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP103:%.*]] = fmul <1 x double> [[BLOCK63]], [[SPLAT_SPLAT65]]
-; CHECK-NEXT:    [[BLOCK66:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP104:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT67:%.*]] = insertelement <1 x double> poison, double [[TMP104]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT68:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT67]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP105:%.*]] = fmul <1 x double> [[BLOCK66]], [[SPLAT_SPLAT68]]
-; CHECK-NEXT:    [[TMP106:%.*]] = fadd <1 x double> [[TMP103]], [[TMP105]]
-; CHECK-NEXT:    [[BLOCK69:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP107:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT70:%.*]] = insertelement <1 x double> poison, double [[TMP107]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT71:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT70]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP108:%.*]] = fmul <1 x double> [[BLOCK69]], [[SPLAT_SPLAT71]]
-; CHECK-NEXT:    [[TMP109:%.*]] = fadd <1 x double> [[TMP106]], [[TMP108]]
-; CHECK-NEXT:    [[TMP110:%.*]] = shufflevector <1 x double> [[TMP109]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP111:%.*]] = shufflevector <3 x double> poison, <3 x double> [[TMP110]], <3 x i32> <i32 3, i32 1, i32 2>
-; CHECK-NEXT:    [[BLOCK72:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP112:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT73:%.*]] = insertelement <1 x double> poison, double [[TMP112]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT74:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT73]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP113:%.*]] = fmul <1 x double> [[BLOCK72]], [[SPLAT_SPLAT74]]
-; CHECK-NEXT:    [[BLOCK75:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP114:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT76:%.*]] = insertelement <1 x double> poison, double [[TMP114]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT77:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT76]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP115:%.*]] = fmul <1 x double> [[BLOCK75]], [[SPLAT_SPLAT77]]
-; CHECK-NEXT:    [[TMP116:%.*]] = fadd <1 x double> [[TMP113]], [[TMP115]]
-; CHECK-NEXT:    [[BLOCK78:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 1>
-; CHECK-NEXT:    [[TMP117:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT79:%.*]] = insertelement <1 x double> poison, double [[TMP117]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT80:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT79]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP118:%.*]] = fmul <1 x double> [[BLOCK78]], [[SPLAT_SPLAT80]]
-; CHECK-NEXT:    [[TMP119:%.*]] = fadd <1 x double> [[TMP116]], [[TMP118]]
-; CHECK-NEXT:    [[TMP120:%.*]] = shufflevector <1 x double> [[TMP119]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP121:%.*]] = shufflevector <3 x double> [[TMP111]], <3 x double> [[TMP120]], <3 x i32> <i32 0, i32 3, i32 2>
-; CHECK-NEXT:    [[BLOCK81:%.*]] = shufflevector <3 x double> [[SPLIT6]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP122:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT82:%.*]] = insertelement <1 x double> poison, double [[TMP122]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT83:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT82]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP123:%.*]] = fmul <1 x double> [[BLOCK81]], [[SPLAT_SPLAT83]]
-; CHECK-NEXT:    [[BLOCK84:%.*]] = shufflevector <3 x double> [[SPLIT7]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP124:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 1
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT85:%.*]] = insertelement <1 x double> poison, double [[TMP124]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT86:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT85]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP125:%.*]] = fmul <1 x double> [[BLOCK84]], [[SPLAT_SPLAT86]]
-; CHECK-NEXT:    [[TMP126:%.*]] = fadd <1 x double> [[TMP123]], [[TMP125]]
-; CHECK-NEXT:    [[BLOCK87:%.*]] = shufflevector <3 x double> [[SPLIT8]], <3 x double> poison, <1 x i32> <i32 2>
-; CHECK-NEXT:    [[TMP127:%.*]] = extractelement <3 x double> [[SPLIT11]], i64 2
-; CHECK-NEXT:    [[SPLAT_SPLATINSERT88:%.*]] = insertelement <1 x double> poison, double [[TMP127]], i64 0
-; CHECK-NEXT:    [[SPLAT_SPLAT89:%.*]] = shufflevector <1 x double> [[SPLAT_SPLATINSERT88]], <1 x double> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP128:%.*]] = fmul <1 x double> [[BLOCK87]], [[SPLAT_SPLAT89]]
-; CHECK-NEXT:    [[TMP129:%.*]] = fadd <1 x double> [[TMP126]], [[TMP128]]
-; CHECK-NEXT:    [[TMP130:%.*]] = shufflevector <1 x double> [[TMP129]], <1 x double> poison, <3 x i32> <i32 0, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP131:%.*]] = shufflevector <3 x double> [[TMP121]], <3 x double> [[TMP130]], <3 x i32> <i32 0, i32 1, i32 3>
-; CHECK-NEXT:    [[TMP132:%.*]] = shufflevector <3 x double> [[TMP71]], <3 x double> [[TMP101]], <6 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5>
-; CHECK-NEXT:    [[TMP133:%.*]] = shufflevector <3 x double> [[TMP131]], <3 x double> poison, <6 x i32> <i32 0, i32 1, i32 2, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT:    [[TMP134:%.*]] = shufflevector <6 x double> [[TMP132]], <6 x double> [[TMP133]], <9 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8>
-; CHECK-NEXT:    ret <9 x double> [[TMP134]]
-;
-
-
-
-entry:
-  br i1 %cond, label %if.then, label %if.else
-
-if.then:                    ; preds = %entry
-  %A.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %A, i32 3, i32 3)
-  br label %if.end
-
-if.else:                                       ; preds = %entry
-  %B.trans = tail call <9 x double> @llvm.matrix.transpose.v9f64(<9 x double> %B, i32 3, i32 3)
-  br label %if.end
-
-if.end:                                        ; preds = %if.then, %if.else
-  %merge = phi <9 x double> [ %A.trans, %if.then], [ %B.trans, %if.else ]
-  %res = tail call <9 x double> @llvm.matrix.multiply.v9f64.v9f64.v9f64(<9 x double> %C, <9 x double> %merge, i32 3, i32 3, i32 3)
-  ret <9 x double> %res
-}
-
 ; Make sure we use a flattened vector when calling @foo and the use its flat vector result properly.
 define <9 x double> @unsupported_call(i1 %cond, <9 x double> %A, <9 x double> %B) {
 ; CHECK-LABEL: @unsupported_call(
-- 
cgit v1.2.3


From a2cee05449636c8e0d630b2ccdc71f2d422227a9 Mon Sep 17 00:00:00 2001
From: Christopher Ferris <cferris1000@users.noreply.github.com>
Date: Wed, 18 Jun 2025 09:12:53 -0700
Subject: [scudo] Make report pointers const. (#144624)

Mark as many of the reportXX functions that take pointers const. This
avoid the need to use const_cast when calling these functions on an
already const pointer.

Fix reportHeaderCorruption calls where an argument was passed into an
append call that didn't use them.
---
 compiler-rt/lib/scudo/standalone/chunk.h    |  2 +-
 compiler-rt/lib/scudo/standalone/combined.h |  2 +-
 compiler-rt/lib/scudo/standalone/report.cpp | 15 +++++++--------
 compiler-rt/lib/scudo/standalone/report.h   | 11 ++++++-----
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/compiler-rt/lib/scudo/standalone/chunk.h b/compiler-rt/lib/scudo/standalone/chunk.h
index a1b8e723d4cb..9da2dc57e71a 100644
--- a/compiler-rt/lib/scudo/standalone/chunk.h
+++ b/compiler-rt/lib/scudo/standalone/chunk.h
@@ -125,7 +125,7 @@ inline void loadHeader(u32 Cookie, const void *Ptr,
   *NewUnpackedHeader = bit_cast<UnpackedHeader>(NewPackedHeader);
   if (UNLIKELY(NewUnpackedHeader->Checksum !=
                computeHeaderChecksum(Cookie, Ptr, NewUnpackedHeader)))
-    reportHeaderCorruption(NewUnpackedHeader, const_cast<void *>(Ptr));
+    reportHeaderCorruption(NewUnpackedHeader, Ptr);
 }
 
 inline bool isValid(u32 Cookie, const void *Ptr,
diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h
index 43655642843c..87acdec2a3ba 100644
--- a/compiler-rt/lib/scudo/standalone/combined.h
+++ b/compiler-rt/lib/scudo/standalone/combined.h
@@ -775,7 +775,7 @@ public:
 
     // Getting the alloc size of a chunk only makes sense if it's allocated.
     if (UNLIKELY(Header.State != Chunk::State::Allocated))
-      reportInvalidChunkState(AllocatorAction::Sizing, const_cast<void *>(Ptr));
+      reportInvalidChunkState(AllocatorAction::Sizing, Ptr);
 
     return getSize(Ptr, &Header);
   }
diff --git a/compiler-rt/lib/scudo/standalone/report.cpp b/compiler-rt/lib/scudo/standalone/report.cpp
index 14a4066d3720..b97a74b078c2 100644
--- a/compiler-rt/lib/scudo/standalone/report.cpp
+++ b/compiler-rt/lib/scudo/standalone/report.cpp
@@ -66,17 +66,16 @@ void NORETURN reportInvalidFlag(const char *FlagType, const char *Value) {
 
 // The checksum of a chunk header is invalid. This could be caused by an
 // {over,under}write of the header, a pointer that is not an actual chunk.
-void NORETURN reportHeaderCorruption(void *Header, void *Ptr) {
+void NORETURN reportHeaderCorruption(void *Header, const void *Ptr) {
   ScopedErrorReport Report;
   Report.append("corrupted chunk header at address %p", Ptr);
   if (*static_cast<Chunk::PackedHeader *>(Header) == 0U) {
     // Header all zero, which could indicate that this might be a pointer that
     // has been double freed but the memory has been released to the kernel.
     Report.append(": chunk header is zero and might indicate memory corruption "
-                  "or a double free\n",
-                  Ptr);
+                  "or a double free\n");
   } else {
-    Report.append(": most likely due to memory corruption\n", Ptr);
+    Report.append(": most likely due to memory corruption\n");
   }
 }
 
@@ -131,13 +130,13 @@ static const char *stringifyAction(AllocatorAction Action) {
 
 // The chunk is not in a state congruent with the operation we want to perform.
 // This is usually the case with a double-free, a realloc of a freed pointer.
-void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr) {
+void NORETURN reportInvalidChunkState(AllocatorAction Action, const void *Ptr) {
   ScopedErrorReport Report;
   Report.append("invalid chunk state when %s address %p\n",
                 stringifyAction(Action), Ptr);
 }
 
-void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr) {
+void NORETURN reportMisalignedPointer(AllocatorAction Action, const void *Ptr) {
   ScopedErrorReport Report;
   Report.append("misaligned pointer when %s address %p\n",
                 stringifyAction(Action), Ptr);
@@ -145,7 +144,7 @@ void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr) {
 
 // The deallocation function used is at odds with the one used to allocate the
 // chunk (eg: new[]/delete or malloc/delete, and so on).
-void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr,
+void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, const void *Ptr,
                                         u8 TypeA, u8 TypeB) {
   ScopedErrorReport Report;
   Report.append("allocation type mismatch when %s address %p (%d vs %d)\n",
@@ -154,7 +153,7 @@ void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr,
 
 // The size specified to the delete operator does not match the one that was
 // passed to new when allocating the chunk.
-void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size,
+void NORETURN reportDeleteSizeMismatch(const void *Ptr, uptr Size,
                                        uptr ExpectedSize) {
   ScopedErrorReport Report;
   Report.append(
diff --git a/compiler-rt/lib/scudo/standalone/report.h b/compiler-rt/lib/scudo/standalone/report.h
index c0214b51560e..c397dd3fc9c6 100644
--- a/compiler-rt/lib/scudo/standalone/report.h
+++ b/compiler-rt/lib/scudo/standalone/report.h
@@ -24,7 +24,7 @@ void NORETURN reportRawError(const char *Message);
 void NORETURN reportInvalidFlag(const char *FlagType, const char *Value);
 
 // Chunk header related errors.
-void NORETURN reportHeaderCorruption(void *Header, void *Ptr);
+void NORETURN reportHeaderCorruption(void *Header, const void *Ptr);
 
 // Sanity checks related error.
 void NORETURN reportSanityCheckError(const char *Field);
@@ -41,11 +41,12 @@ enum class AllocatorAction : u8 {
   Reallocating,
   Sizing,
 };
-void NORETURN reportInvalidChunkState(AllocatorAction Action, void *Ptr);
-void NORETURN reportMisalignedPointer(AllocatorAction Action, void *Ptr);
-void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, void *Ptr,
+void NORETURN reportInvalidChunkState(AllocatorAction Action, const void *Ptr);
+void NORETURN reportMisalignedPointer(AllocatorAction Action, const void *Ptr);
+void NORETURN reportDeallocTypeMismatch(AllocatorAction Action, const void *Ptr,
                                         u8 TypeA, u8 TypeB);
-void NORETURN reportDeleteSizeMismatch(void *Ptr, uptr Size, uptr ExpectedSize);
+void NORETURN reportDeleteSizeMismatch(const void *Ptr, uptr Size,
+                                       uptr ExpectedSize);
 
 // C wrappers errors.
 void NORETURN reportAlignmentNotPowerOfTwo(uptr Alignment);
-- 
cgit v1.2.3


From 13510c07364dc3ac30f34e73c98ac8dc75e7efc7 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Wed, 18 Jun 2025 12:13:50 -0400
Subject: [libc++] Make list constexpr as part of P3372R3 (#129799)

This patch makes `std::list` constexpr as part of P3372R3.

Fixes #128659.
---
 libcxx/docs/FeatureTestMacroTable.rst              |   2 +
 libcxx/include/list                                | 552 ++++++++++--------
 libcxx/include/version                             |   2 +
 .../std/containers/sequences/list/compare.pass.cpp |  24 +-
 .../sequences/list/compare.three_way.pass.cpp      |   6 +-
 .../sequences/list/get_allocator.pass.cpp          |  13 +-
 .../sequences/list/incomplete_type.pass.cpp        |  13 +-
 .../containers/sequences/list/iterators.pass.cpp   |  27 +-
 .../sequences/list/list.capacity/empty.pass.cpp    |  13 +-
 .../sequences/list/list.capacity/max_size.pass.cpp |  13 +-
 .../list/list.capacity/resize_size.pass.cpp        |  62 +-
 .../list/list.capacity/resize_size_value.pass.cpp  |  13 +-
 .../sequences/list/list.capacity/size.pass.cpp     |  13 +-
 .../sequences/list/list.cons/assign_copy.pass.cpp  |  13 +-
 .../list.cons/assign_initializer_list.pass.cpp     |  13 +-
 .../sequences/list/list.cons/assign_move.pass.cpp  |  13 +-
 .../sequences/list/list.cons/copy.pass.cpp         |  13 +-
 .../sequences/list/list.cons/copy_alloc.pass.cpp   |  13 +-
 .../sequences/list/list.cons/default.pass.cpp      |  13 +-
 .../list/list.cons/default_stack_alloc.pass.cpp    |  13 +-
 .../sequences/list/list.cons/from_range.pass.cpp   |  20 +-
 .../list/list.cons/initializer_list.pass.cpp       |  13 +-
 .../list/list.cons/initializer_list_alloc.pass.cpp |  13 +-
 .../list/list.cons/input_iterator.pass.cpp         |  27 +-
 .../sequences/list/list.cons/move.pass.cpp         |  13 +-
 .../sequences/list/list.cons/move_alloc.pass.cpp   |  13 +-
 .../list.cons/op_equal_initializer_list.pass.cpp   |  13 +-
 .../sequences/list/list.cons/size_type.pass.cpp    |  39 +-
 .../list/list.cons/size_value_alloc.pass.cpp       |  13 +-
 .../sequences/list/list.erasure/erase.pass.cpp     |  27 +-
 .../sequences/list/list.erasure/erase_if.pass.cpp  |  27 +-
 .../list/list.modifiers/append_range.pass.cpp      |  20 +-
 .../list/list.modifiers/assign_range.pass.cpp      |  20 +-
 .../sequences/list/list.modifiers/clear.pass.cpp   |  13 +-
 .../sequences/list/list.modifiers/emplace.pass.cpp |  19 +-
 .../list/list.modifiers/emplace_back.pass.cpp      |  19 +-
 .../list/list.modifiers/emplace_front.pass.cpp     |  19 +-
 .../list/list.modifiers/erase_iter.pass.cpp        |  13 +-
 .../list/list.modifiers/erase_iter_iter.pass.cpp   |  13 +-
 .../insert_iter_initializer_list.pass.cpp          |  13 +-
 .../list.modifiers/insert_iter_iter_iter.pass.cpp  |  59 +-
 .../list.modifiers/insert_iter_rvalue.pass.cpp     |  13 +-
 .../list.modifiers/insert_iter_size_value.pass.cpp |  33 +-
 .../list/list.modifiers/insert_iter_value.pass.cpp |  33 +-
 .../list/list.modifiers/insert_range.pass.cpp      |  23 +-
 .../list/list.modifiers/pop_back.pass.cpp          |  13 +-
 .../list/list.modifiers/pop_front.pass.cpp         |  13 +-
 .../list/list.modifiers/prepend_range.pass.cpp     |  20 +-
 .../list/list.modifiers/push_back.pass.cpp         |  13 +-
 .../list/list.modifiers/push_back_rvalue.pass.cpp  |  13 +-
 .../list/list.modifiers/push_front.pass.cpp        |  13 +-
 .../list/list.modifiers/push_front_rvalue.pass.cpp |  13 +-
 .../sequences/list/list.ops/merge.pass.cpp         |  13 +-
 .../sequences/list/list.ops/merge_comp.pass.cpp    |  13 +-
 .../sequences/list/list.ops/remove.pass.cpp        |  25 +-
 .../sequences/list/list.ops/remove_if.pass.cpp     |  25 +-
 .../sequences/list/list.ops/reverse.pass.cpp       |  13 +-
 .../sequences/list/list.ops/sort.pass.cpp          |  19 +-
 .../sequences/list/list.ops/sort_comp.pass.cpp     |  57 +-
 .../list/list.ops/splice_pos_list.pass.cpp         |  13 +-
 .../list/list.ops/splice_pos_list_iter.pass.cpp    |  13 +-
 .../list.ops/splice_pos_list_iter_iter.pass.cpp    |  13 +-
 .../sequences/list/list.ops/unique.pass.cpp        |  13 +-
 .../sequences/list/list.ops/unique_pred.pass.cpp   |  23 +-
 .../sequences/list/list.special/swap.pass.cpp      |  13 +-
 .../list/list.special/swap_noexcept.pass.cpp       |  13 +-
 .../list.version.compile.pass.cpp                  |  27 +
 .../version.version.compile.pass.cpp               |  27 +
 libcxx/test/support/min_allocator.h                | 647 ++++++++++-----------
 .../generate_feature_test_macro_components.py      |   5 +
 70 files changed, 1600 insertions(+), 842 deletions(-)

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 3e6fd643f620..5ebc9bb7dcda 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -422,6 +422,8 @@ Status
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_forward_list``                       ``202502L``
     ---------------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_list``                               ``202502L``
+    ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_new``                                ``202406L``
     ---------------------------------------------------------- -----------------
     ``__cpp_lib_constexpr_queue``                              ``202502L``
diff --git a/libcxx/include/list b/libcxx/include/list
index 98610f59ed74..2896231203d9 100644
--- a/libcxx/include/list
+++ b/libcxx/include/list
@@ -297,14 +297,20 @@ struct __list_node_base {
   __base_pointer __prev_;
   __base_pointer __next_;
 
-  _LIBCPP_HIDE_FROM_ABI __list_node_base() : __prev_(__self()), __next_(__self()) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_node_base() : __prev_(__self()), __next_(__self()) {}
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
   _LIBCPP_HIDE_FROM_ABI explicit __list_node_base(__base_pointer __prev, __base_pointer __next)
       : __prev_(__prev), __next_(__next) {}
 
-  _LIBCPP_HIDE_FROM_ABI __base_pointer __self() { return pointer_traits<__base_pointer>::pointer_to(*this); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __base_pointer __self() {
+    return pointer_traits<__base_pointer>::pointer_to(*this);
+  }
 
-  _LIBCPP_HIDE_FROM_ABI __node_pointer __as_node() { return static_cast<__node_pointer>(__self()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer __as_node() {
+    return pointer_traits<__node_pointer>::pointer_to(
+        *static_cast<typename pointer_traits<__node_pointer>::element_type*>(this));
+  }
 };
 
 template <class _Tp, class _VoidPtr>
@@ -319,7 +325,7 @@ private:
   };
 
 public:
-  _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI _Tp& __get_value() { return __value_; }
 #  else
 
 private:
@@ -332,10 +338,14 @@ public:
   typedef __list_node_base<_Tp, _VoidPtr> __base;
   typedef typename __base::__base_pointer __base_pointer;
 
-  _LIBCPP_HIDE_FROM_ABI explicit __list_node(__base_pointer __prev, __base_pointer __next) : __base(__prev, __next) {}
-  _LIBCPP_HIDE_FROM_ABI ~__list_node() {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __list_node(__base_pointer __prev, __base_pointer __next)
+      : __base(__prev, __next) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI ~__list_node() {}
 
-  _LIBCPP_HIDE_FROM_ABI __base_pointer __as_link() { return __base::__self(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __base_pointer __as_link() {
+    return pointer_traits<__base_pointer>::pointer_to(
+        *static_cast<typename pointer_traits<__base_pointer>::element_type*>(std::addressof(*this)));
+  }
 };
 
 template <class _Tp, class _Alloc = allocator<_Tp> >
@@ -352,7 +362,8 @@ class __list_iterator {
 
   __base_pointer __ptr_;
 
-  _LIBCPP_HIDE_FROM_ABI explicit __list_iterator(__base_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __list_iterator(__base_pointer __p) _NOEXCEPT
+      : __ptr_(__p) {}
 
   template <class, class>
   friend class list;
@@ -368,37 +379,41 @@ public:
   typedef __rebind_pointer_t<_VoidPtr, value_type> pointer;
   typedef typename pointer_traits<pointer>::difference_type difference_type;
 
-  _LIBCPP_HIDE_FROM_ABI __list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __ptr_->__as_node()->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    return __ptr_->__as_node()->__get_value();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
     return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
   }
 
-  _LIBCPP_HIDE_FROM_ABI __list_iterator& operator++() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_iterator& operator++() {
     __ptr_ = __ptr_->__next_;
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __list_iterator operator++(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_iterator operator++(int) {
     __list_iterator __t(*this);
     ++(*this);
     return __t;
   }
 
-  _LIBCPP_HIDE_FROM_ABI __list_iterator& operator--() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_iterator& operator--() {
     __ptr_ = __ptr_->__prev_;
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __list_iterator operator--(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_iterator operator--(int) {
     __list_iterator __t(*this);
     --(*this);
     return __t;
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __list_iterator& __x, const __list_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __list_iterator& __x, const __list_iterator& __y) {
     return __x.__ptr_ == __y.__ptr_;
   }
-  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __list_iterator& __x, const __list_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __list_iterator& __x, const __list_iterator& __y) {
     return !(__x == __y);
   }
 };
@@ -410,7 +425,8 @@ class __list_const_iterator {
 
   __base_pointer __ptr_;
 
-  _LIBCPP_HIDE_FROM_ABI explicit __list_const_iterator(__base_pointer __p) _NOEXCEPT : __ptr_(__p) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __list_const_iterator(__base_pointer __p) _NOEXCEPT
+      : __ptr_(__p) {}
 
   template <class, class>
   friend class list;
@@ -424,39 +440,43 @@ public:
   typedef __rebind_pointer_t<_VoidPtr, const value_type> pointer;
   typedef typename pointer_traits<pointer>::difference_type difference_type;
 
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator(const __list_iterator<_Tp, _VoidPtr>& __p) _NOEXCEPT
-      : __ptr_(__p.__ptr_) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_const_iterator() _NOEXCEPT : __ptr_(nullptr) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  __list_const_iterator(const __list_iterator<_Tp, _VoidPtr>& __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
 
-  _LIBCPP_HIDE_FROM_ABI reference operator*() const { return __ptr_->__as_node()->__get_value(); }
-  _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
+    return __ptr_->__as_node()->__get_value();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
     return pointer_traits<pointer>::pointer_to(__ptr_->__as_node()->__get_value());
   }
 
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator++() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator++() {
     __ptr_ = __ptr_->__next_;
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator++(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator++(int) {
     __list_const_iterator __t(*this);
     ++(*this);
     return __t;
   }
 
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator--() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_const_iterator& operator--() {
     __ptr_ = __ptr_->__prev_;
     return *this;
   }
-  _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator--(int) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_const_iterator operator--(int) {
     __list_const_iterator __t(*this);
     --(*this);
     return __t;
   }
 
-  friend _LIBCPP_HIDE_FROM_ABI bool operator==(const __list_const_iterator& __x, const __list_const_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator==(const __list_const_iterator& __x, const __list_const_iterator& __y) {
     return __x.__ptr_ == __y.__ptr_;
   }
-  friend _LIBCPP_HIDE_FROM_ABI bool operator!=(const __list_const_iterator& __x, const __list_const_iterator& __y) {
+  friend _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool
+  operator!=(const __list_const_iterator& __x, const __list_const_iterator& __y) {
     return !(__x == __y);
   }
 };
@@ -497,43 +517,49 @@ protected:
   __node_base __end_;
   _LIBCPP_COMPRESSED_PAIR(size_type, __size_, __node_allocator, __node_alloc_);
 
-  _LIBCPP_HIDE_FROM_ABI __base_pointer __end_as_link() const _NOEXCEPT {
-    return const_cast<__node_base&>(__end_).__self();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __base_pointer __end_as_link() const _NOEXCEPT {
+    return pointer_traits<__base_pointer>::pointer_to(const_cast<__node_base&>(__end_));
   }
 
-  _LIBCPP_HIDE_FROM_ABI size_type __node_alloc_max_size() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type __node_alloc_max_size() const _NOEXCEPT {
     return __node_alloc_traits::max_size(__node_alloc_);
   }
-  _LIBCPP_HIDE_FROM_ABI static void __unlink_nodes(__base_pointer __f, __base_pointer __l) _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI static void
+  __unlink_nodes(__base_pointer __f, __base_pointer __l) _NOEXCEPT;
 
-  _LIBCPP_HIDE_FROM_ABI __list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value);
-  _LIBCPP_HIDE_FROM_ABI __list_imp(const allocator_type& __a);
-  _LIBCPP_HIDE_FROM_ABI __list_imp(const __node_allocator& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_imp()
+      _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_imp(const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_imp(const __node_allocator& __a);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI __list_imp(__node_allocator&& __a) _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __list_imp(__node_allocator&& __a) _NOEXCEPT;
 #  endif
-  _LIBCPP_HIDE_FROM_ABI ~__list_imp();
-  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
-  _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __size_ == 0; }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI ~__list_imp();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __size_ == 0; }
 
-  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__end_.__next_); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return const_iterator(__end_.__next_); }
-  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(__end_as_link()); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return const_iterator(__end_as_link()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return iterator(__end_.__next_); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT {
+    return const_iterator(__end_.__next_);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return iterator(__end_as_link()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT {
+    return const_iterator(__end_as_link());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI void swap(__list_imp& __c)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void swap(__list_imp& __c)
 #  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT;
 #  else
       _NOEXCEPT_(!__alloc_traits::propagate_on_container_swap::value || __is_nothrow_swappable_v<allocator_type>);
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c) {
     __copy_assign_alloc(
         __c, integral_constant<bool, __node_alloc_traits::propagate_on_container_copy_assignment::value>());
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c)
       _NOEXCEPT_(!__node_alloc_traits::propagate_on_container_move_assignment::value ||
                  is_nothrow_move_assignable<__node_allocator>::value) {
     __move_assign_alloc(
@@ -541,7 +567,8 @@ protected:
   }
 
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI __node_pointer __create_node(__base_pointer __prev, __base_pointer __next, _Args&&... __args) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer
+  __create_node(__base_pointer __prev, __base_pointer __next, _Args&&... __args) {
     __allocation_guard<__node_allocator> __guard(__node_alloc_, 1);
     // Begin the lifetime of the node itself. Note that this doesn't begin the lifetime of the value
     // held inside the node, since we need to use the allocator's construct() method for that.
@@ -557,7 +584,7 @@ protected:
     return __guard.__release_ptr();
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __delete_node(__node_pointer __node) {
     // For the same reason as above, we use the allocator's destroy() method for the value_type,
     // but not for the node itself.
     __node_alloc_traits::destroy(__node_alloc_, std::addressof(__node->__get_value()));
@@ -566,54 +593,57 @@ protected:
   }
 
 private:
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c, true_type) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp& __c, true_type) {
     if (__node_alloc_ != __c.__node_alloc_)
       clear();
     __node_alloc_ = __c.__node_alloc_;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp&, false_type) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __copy_assign_alloc(const __list_imp&, false_type) {}
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c, true_type)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp& __c, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value) {
     __node_alloc_ = std::move(__c.__node_alloc_);
   }
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp&, false_type) _NOEXCEPT {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign_alloc(__list_imp&, false_type) _NOEXCEPT {}
 };
 
 // Unlink nodes [__f, __l]
 template <class _Tp, class _Alloc>
-inline void __list_imp<_Tp, _Alloc>::__unlink_nodes(__base_pointer __f, __base_pointer __l) _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void
+__list_imp<_Tp, _Alloc>::__unlink_nodes(__base_pointer __f, __base_pointer __l) _NOEXCEPT {
   __f->__prev_->__next_ = __l->__next_;
   __l->__next_->__prev_ = __f->__prev_;
 }
 
 template <class _Tp, class _Alloc>
-inline __list_imp<_Tp, _Alloc>::__list_imp() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __list_imp<_Tp, _Alloc>::__list_imp()
+    _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value)
     : __size_(0) {}
 
 template <class _Tp, class _Alloc>
-inline __list_imp<_Tp, _Alloc>::__list_imp(const allocator_type& __a)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __list_imp<_Tp, _Alloc>::__list_imp(const allocator_type& __a)
     : __size_(0), __node_alloc_(__node_allocator(__a)) {}
 
 template <class _Tp, class _Alloc>
-inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a) : __size_(0), __node_alloc_(__a) {}
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __list_imp<_Tp, _Alloc>::__list_imp(const __node_allocator& __a)
+    : __size_(0), __node_alloc_(__a) {}
 
 #  ifndef _LIBCPP_CXX03_LANG
 template <class _Tp, class _Alloc>
-inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline __list_imp<_Tp, _Alloc>::__list_imp(__node_allocator&& __a) _NOEXCEPT
     : __size_(0),
       __node_alloc_(std::move(__a)) {}
 #  endif
 
 template <class _Tp, class _Alloc>
-__list_imp<_Tp, _Alloc>::~__list_imp() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 __list_imp<_Tp, _Alloc>::~__list_imp() {
   clear();
 }
 
 template <class _Tp, class _Alloc>
-void __list_imp<_Tp, _Alloc>::clear() _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void __list_imp<_Tp, _Alloc>::clear() _NOEXCEPT {
   if (!empty()) {
     __base_pointer __f = __end_.__next_;
     __base_pointer __l = __end_as_link();
@@ -628,7 +658,7 @@ void __list_imp<_Tp, _Alloc>::clear() _NOEXCEPT {
 }
 
 template <class _Tp, class _Alloc>
-void __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void __list_imp<_Tp, _Alloc>::swap(__list_imp& __c)
 #  if _LIBCPP_STD_VER >= 14
     _NOEXCEPT
 #  else
@@ -686,170 +716,204 @@ public:
   typedef void __remove_return_type;
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI list() _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {}
-  _LIBCPP_HIDE_FROM_ABI explicit list(const allocator_type& __a) : __base(__a) {}
-  _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list()
+      _NOEXCEPT_(is_nothrow_default_constructible<__node_allocator>::value) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit list(const allocator_type& __a) : __base(__a) {}
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n);
 #  if _LIBCPP_STD_VER >= 14
-  _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit list(size_type __n, const allocator_type& __a);
 #  endif
-  _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x);
   template <__enable_if_t<__is_allocator<_Alloc>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI list(size_type __n, const value_type& __x, const allocator_type& __a) : __base(__a) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  list(size_type __n, const value_type& __x, const allocator_type& __a)
+      : __base(__a) {
     for (; __n > 0; --__n)
       push_back(__x);
   }
 
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l);
 
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l, const allocator_type& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(_InpIter __f, _InpIter __l, const allocator_type& __a);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  list(from_range_t, _Range&& __range, const allocator_type& __a = allocator_type())
       : __base(__a) {
     prepend_range(std::forward<_Range>(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI list(const list& __c);
-  _LIBCPP_HIDE_FROM_ABI list(const list& __c, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_HIDE_FROM_ABI list& operator=(const list& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(const list& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  list(const list& __c, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list& operator=(const list& __c);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il);
-  _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il, const allocator_type& __a);
-
-  _LIBCPP_HIDE_FROM_ABI list(list&& __c) _NOEXCEPT_(is_nothrow_move_constructible<__node_allocator>::value);
-  _LIBCPP_HIDE_FROM_ABI list(list&& __c, const __type_identity_t<allocator_type>& __a);
-  _LIBCPP_HIDE_FROM_ABI list& operator=(list&& __c) noexcept(
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(initializer_list<value_type> __il);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI
+  list(initializer_list<value_type> __il, const allocator_type& __a);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(list&& __c)
+      _NOEXCEPT_(is_nothrow_move_constructible<__node_allocator>::value);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list(list&& __c, const __type_identity_t<allocator_type>& __a);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list& operator=(list&& __c) noexcept(
       (__node_alloc_traits::propagate_on_container_move_assignment::value &&
        is_nothrow_move_assignable<__node_allocator>::value) ||
       allocator_traits<allocator_type>::is_always_equal::value);
 
-  _LIBCPP_HIDE_FROM_ABI list& operator=(initializer_list<value_type> __il) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI list& operator=(initializer_list<value_type> __il) {
     assign(__il.begin(), __il.end());
     return *this;
   }
 
-  _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) { assign(__il.begin(), __il.end()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(initializer_list<value_type> __il) {
+    assign(__il.begin(), __il.end());
+  }
 #  endif // _LIBCPP_CXX03_LANG
 
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI void assign(_InpIter __f, _InpIter __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(_InpIter __f, _InpIter __l);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign_range(_Range&& __range) {
     __assign_with_sentinel(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void assign(size_type __n, const value_type& __x);
 
-  _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI allocator_type get_allocator() const _NOEXCEPT;
 
-  _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return this->__size_; }
-  [[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT { return __base::empty(); }
-  _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return this->__size_; }
+  [[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool empty() const _NOEXCEPT {
+    return __base::empty();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI size_type max_size() const _NOEXCEPT {
     return std::min<size_type>(this->__node_alloc_max_size(), numeric_limits<difference_type >::max());
   }
 
-  _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __base::begin(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __base::begin(); }
-  _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __base::end(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __base::end(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT { return __base::begin(); }
-  _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __base::end(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator begin() _NOEXCEPT { return __base::begin(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator begin() const _NOEXCEPT { return __base::begin(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator end() _NOEXCEPT { return __base::end(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator end() const _NOEXCEPT { return __base::end(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cbegin() const _NOEXCEPT {
+    return __base::begin();
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_iterator cend() const _NOEXCEPT { return __base::end(); }
 
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT { return reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT { return reverse_iterator(begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT { return const_reverse_iterator(end()); }
-  _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT { return const_reverse_iterator(begin()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rbegin() _NOEXCEPT {
+    return reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reverse_iterator rend() _NOEXCEPT {
+    return reverse_iterator(begin());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator rend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crbegin() const _NOEXCEPT {
+    return const_reverse_iterator(end());
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reverse_iterator crend() const _NOEXCEPT {
+    return const_reverse_iterator(begin());
+  }
 
-  _LIBCPP_HIDE_FROM_ABI reference front() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference front() {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
     return __base::__end_.__next_->__as_node()->__get_value();
   }
-  _LIBCPP_HIDE_FROM_ABI const_reference front() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference front() const {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::front called on empty list");
     return __base::__end_.__next_->__as_node()->__get_value();
   }
-  _LIBCPP_HIDE_FROM_ABI reference back() {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference back() {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
     return __base::__end_.__prev_->__as_node()->__get_value();
   }
-  _LIBCPP_HIDE_FROM_ABI const_reference back() const {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI const_reference back() const {
     _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::back called on empty list");
     return __base::__end_.__prev_->__as_node()->__get_value();
   }
 
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __x);
-  _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_front(value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_back(value_type&& __x);
 
 #    if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void prepend_range(_Range&& __range) {
     insert_range(begin(), std::forward<_Range>(__range));
   }
 
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void append_range(_Range&& __range) {
     insert_range(end(), std::forward<_Range>(__range));
   }
 #    endif
 
   template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
 #    if _LIBCPP_STD_VER >= 17
-  _LIBCPP_HIDE_FROM_ABI reference emplace_front(_Args&&... __args);
+      _LIBCPP_HIDE_FROM_ABI reference
+      emplace_front(_Args&&... __args);
 #    else
-  _LIBCPP_HIDE_FROM_ABI void emplace_front(_Args&&... __args);
+      _LIBCPP_HIDE_FROM_ABI void
+      emplace_front(_Args&&... __args);
 #    endif
   template <class... _Args>
+  _LIBCPP_CONSTEXPR_SINCE_CXX26
 #    if _LIBCPP_STD_VER >= 17
-  _LIBCPP_HIDE_FROM_ABI reference emplace_back(_Args&&... __args);
+      _LIBCPP_HIDE_FROM_ABI reference
+      emplace_back(_Args&&... __args);
 #    else
-  _LIBCPP_HIDE_FROM_ABI void emplace_back(_Args&&... __args);
+      _LIBCPP_HIDE_FROM_ABI void
+      emplace_back(_Args&&... __args);
 #    endif
   template <class... _Args>
-  _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __p, _Args&&... __args);
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, value_type&& __x);
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, initializer_list<value_type> __il) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __p, initializer_list<value_type> __il) {
     return insert(__p, __il.begin(), __il.end());
   }
 #  endif // _LIBCPP_CXX03_LANG
 
-  _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __x);
-  _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_front(const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void push_back(const value_type& __x);
 
 #  ifndef _LIBCPP_CXX03_LANG
   template <class _Arg>
-  _LIBCPP_HIDE_FROM_ABI void __emplace_back(_Arg&& __arg) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __emplace_back(_Arg&& __arg) {
     emplace_back(std::forward<_Arg>(__arg));
   }
 #  else
   _LIBCPP_HIDE_FROM_ABI void __emplace_back(value_type const& __arg) { push_back(__arg); }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x);
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, size_type __n, const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert(const_iterator __p, size_type __n, const value_type& __x);
 
   template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> = 0>
-  _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InpIter __f, _InpIter __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __p, _InpIter __f, _InpIter __l);
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<_Tp> _Range>
-  _LIBCPP_HIDE_FROM_ABI iterator insert_range(const_iterator __position, _Range&& __range) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  insert_range(const_iterator __position, _Range&& __range) {
     return __insert_with_sentinel(__position, ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI void swap(list& __c)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void swap(list& __c)
 #  if _LIBCPP_STD_VER >= 14
       _NOEXCEPT
 #  else
@@ -858,72 +922,80 @@ public:
   {
     __base::swap(__c);
   }
-  _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __base::clear(); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT { __base::clear(); }
 
-  _LIBCPP_HIDE_FROM_ABI void pop_front();
-  _LIBCPP_HIDE_FROM_ABI void pop_back();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void pop_front();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void pop_back();
 
-  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
-  _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __p);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __f, const_iterator __l);
 
-  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
-  _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void resize(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void resize(size_type __n, const value_type& __x);
 
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c) { splice(__p, __c); }
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __i) { splice(__p, __c, __i); }
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __f, const_iterator __l) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c) { splice(__p, __c); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list&& __c, const_iterator __i) {
+    splice(__p, __c, __i);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  splice(const_iterator __p, list&& __c, const_iterator __f, const_iterator __l) {
     splice(__p, __c, __f, __l);
   }
 #  endif
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __i);
-  _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void splice(const_iterator __p, list& __c, const_iterator __i);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l);
 
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type remove(const value_type& __x);
   template <class _Pred>
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Pred __pred);
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type remove_if(_Pred __pred);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type unique() { return unique(__equal_to()); }
   template <class _BinaryPred>
-  _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPred __binary_pred);
-  _LIBCPP_HIDE_FROM_ABI void merge(list& __c);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __remove_return_type unique(_BinaryPred __binary_pred);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(list& __c);
 #  ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI void merge(list&& __c) { merge(__c); }
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(list&& __c) { merge(__c); }
 
   template <class _Comp>
-  _LIBCPP_HIDE_FROM_ABI void merge(list&& __c, _Comp __comp) {
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(list&& __c, _Comp __comp) {
     merge(__c, __comp);
   }
 #  endif
   template <class _Comp>
-  _LIBCPP_HIDE_FROM_ABI void merge(list& __c, _Comp __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void merge(list& __c, _Comp __comp);
 
-  _LIBCPP_HIDE_FROM_ABI void sort();
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void sort();
   template <class _Comp>
-  _LIBCPP_HIDE_FROM_ABI void sort(_Comp __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void sort(_Comp __comp);
 
-  _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void reverse() _NOEXCEPT;
 
-  _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI bool __invariants() const;
 
 private:
   template <class _Iterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __f, _Sentinel __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __f, _Sentinel __l);
 
   template <class _Iterator, class _Sentinel>
-  _LIBCPP_HIDE_FROM_ABI iterator __insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l);
-
-  _LIBCPP_HIDE_FROM_ABI static void __link_nodes(__base_pointer __p, __base_pointer __f, __base_pointer __l);
-  _LIBCPP_HIDE_FROM_ABI void __link_nodes_at_front(__base_pointer __f, __base_pointer __l);
-  _LIBCPP_HIDE_FROM_ABI void __link_nodes_at_back(__base_pointer __f, __base_pointer __l);
-  _LIBCPP_HIDE_FROM_ABI iterator __iterator(size_type __n);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator
+  __insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l);
+
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI static void
+  __link_nodes(__base_pointer __p, __base_pointer __f, __base_pointer __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+  __link_nodes_at_front(__base_pointer __f, __base_pointer __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __link_nodes_at_back(__base_pointer __f, __base_pointer __l);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI iterator __iterator(size_type __n);
   // TODO: Make this _LIBCPP_HIDE_FROM_ABI
   template <class _Comp>
-  _LIBCPP_HIDDEN static iterator __sort(iterator __f1, iterator __e2, size_type __n, _Comp& __comp);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDDEN static iterator
+  __sort(iterator __f1, iterator __e2, size_type __n, _Comp& __comp);
 
-  _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, true_type)
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, true_type)
       _NOEXCEPT_(is_nothrow_move_assignable<__node_allocator>::value);
-  _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, false_type);
+  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void __move_assign(list& __c, false_type);
 };
 
 #  if _LIBCPP_STD_VER >= 17
@@ -949,7 +1021,8 @@ list(from_range_t, _Range&&, _Alloc = _Alloc()) -> list<ranges::range_value_t<_R
 
 // Link in nodes [__f, __l] just prior to __p
 template <class _Tp, class _Alloc>
-inline void list<_Tp, _Alloc>::__link_nodes(__base_pointer __p, __base_pointer __f, __base_pointer __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void
+list<_Tp, _Alloc>::__link_nodes(__base_pointer __p, __base_pointer __f, __base_pointer __l) {
   __p->__prev_->__next_ = __f;
   __f->__prev_          = __p->__prev_;
   __p->__prev_          = __l;
@@ -958,7 +1031,8 @@ inline void list<_Tp, _Alloc>::__link_nodes(__base_pointer __p, __base_pointer _
 
 // Link in nodes [__f, __l] at the front of the list
 template <class _Tp, class _Alloc>
-inline void list<_Tp, _Alloc>::__link_nodes_at_front(__base_pointer __f, __base_pointer __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void
+list<_Tp, _Alloc>::__link_nodes_at_front(__base_pointer __f, __base_pointer __l) {
   __f->__prev_           = __base::__end_as_link();
   __l->__next_           = __base::__end_.__next_;
   __l->__next_->__prev_  = __l;
@@ -967,7 +1041,8 @@ inline void list<_Tp, _Alloc>::__link_nodes_at_front(__base_pointer __f, __base_
 
 // Link in nodes [__f, __l] at the back of the list
 template <class _Tp, class _Alloc>
-inline void list<_Tp, _Alloc>::__link_nodes_at_back(__base_pointer __f, __base_pointer __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void
+list<_Tp, _Alloc>::__link_nodes_at_back(__base_pointer __f, __base_pointer __l) {
   __l->__next_           = __base::__end_as_link();
   __f->__prev_           = __base::__end_.__prev_;
   __f->__prev_->__next_  = __f;
@@ -975,12 +1050,12 @@ inline void list<_Tp, _Alloc>::__link_nodes_at_back(__base_pointer __f, __base_p
 }
 
 template <class _Tp, class _Alloc>
-inline typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::__iterator(size_type __n) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::__iterator(size_type __n) {
   return __n <= this->__size_ / 2 ? std::next(begin(), __n) : std::prev(end(), this->__size_ - __n);
 }
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(size_type __n) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(size_type __n) {
   for (; __n > 0; --__n)
 #  ifndef _LIBCPP_CXX03_LANG
     emplace_back();
@@ -991,41 +1066,43 @@ list<_Tp, _Alloc>::list(size_type __n) {
 
 #  if _LIBCPP_STD_VER >= 14
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(size_type __n, const allocator_type& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(size_type __n, const allocator_type& __a) : __base(__a) {
   for (; __n > 0; --__n)
     emplace_back();
 }
 #  endif
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(size_type __n, const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(size_type __n, const value_type& __x) {
   for (; __n > 0; --__n)
     push_back(__x);
 }
 
 template <class _Tp, class _Alloc>
 template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
-list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l) {
   for (; __f != __l; ++__f)
     __emplace_back(*__f);
 }
 
 template <class _Tp, class _Alloc>
 template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
-list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l, const allocator_type& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(_InpIter __f, _InpIter __l, const allocator_type& __a)
+    : __base(__a) {
   for (; __f != __l; ++__f)
     __emplace_back(*__f);
 }
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(const list& __c)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(const list& __c)
     : __base(__node_alloc_traits::select_on_container_copy_construction(__c.__node_alloc_)) {
   for (const_iterator __i = __c.begin(), __e = __c.end(); __i != __e; ++__i)
     push_back(*__i);
 }
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(const list& __c, const __type_identity_t<allocator_type>& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(const list& __c, const __type_identity_t<allocator_type>& __a)
+    : __base(__a) {
   for (const_iterator __i = __c.begin(), __e = __c.end(); __i != __e; ++__i)
     push_back(*__i);
 }
@@ -1033,25 +1110,28 @@ list<_Tp, _Alloc>::list(const list& __c, const __type_identity_t<allocator_type>
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(initializer_list<value_type> __il, const allocator_type& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(initializer_list<value_type> __il, const allocator_type& __a)
+    : __base(__a) {
   for (typename initializer_list<value_type>::const_iterator __i = __il.begin(), __e = __il.end(); __i != __e; ++__i)
     push_back(*__i);
 }
 
 template <class _Tp, class _Alloc>
-list<_Tp, _Alloc>::list(initializer_list<value_type> __il) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 list<_Tp, _Alloc>::list(initializer_list<value_type> __il) {
   for (typename initializer_list<value_type>::const_iterator __i = __il.begin(), __e = __il.end(); __i != __e; ++__i)
     push_back(*__i);
 }
 
 template <class _Tp, class _Alloc>
-inline list<_Tp, _Alloc>::list(list&& __c) noexcept(is_nothrow_move_constructible<__node_allocator>::value)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline list<_Tp, _Alloc>::list(list&& __c) noexcept(
+    is_nothrow_move_constructible<__node_allocator>::value)
     : __base(std::move(__c.__node_alloc_)) {
   splice(end(), __c);
 }
 
 template <class _Tp, class _Alloc>
-inline list<_Tp, _Alloc>::list(list&& __c, const __type_identity_t<allocator_type>& __a) : __base(__a) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline list<_Tp, _Alloc>::list(list&& __c, const __type_identity_t<allocator_type>& __a)
+    : __base(__a) {
   if (__a == __c.get_allocator())
     splice(end(), __c);
   else {
@@ -1061,7 +1141,7 @@ inline list<_Tp, _Alloc>::list(list&& __c, const __type_identity_t<allocator_typ
 }
 
 template <class _Tp, class _Alloc>
-inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(list&& __c) noexcept(
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(list&& __c) noexcept(
     (__node_alloc_traits::propagate_on_container_move_assignment::value &&
      is_nothrow_move_assignable<__node_allocator>::value) ||
     allocator_traits<allocator_type>::is_always_equal::value) {
@@ -1070,7 +1150,7 @@ inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(list&& __c) noexcept(
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::__move_assign(list& __c, false_type) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::__move_assign(list& __c, false_type) {
   if (this->__node_alloc_ != __c.__node_alloc_) {
     typedef move_iterator<iterator> _Ip;
     assign(_Ip(__c.begin()), _Ip(__c.end()));
@@ -1079,8 +1159,8 @@ void list<_Tp, _Alloc>::__move_assign(list& __c, false_type) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::__move_assign(list& __c,
-                                      true_type) noexcept(is_nothrow_move_assignable<__node_allocator>::value) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void
+list<_Tp, _Alloc>::__move_assign(list& __c, true_type) noexcept(is_nothrow_move_assignable<__node_allocator>::value) {
   clear();
   __base::__move_assign_alloc(__c);
   splice(end(), __c);
@@ -1089,7 +1169,7 @@ void list<_Tp, _Alloc>::__move_assign(list& __c,
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(const list& __c) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(const list& __c) {
   if (this != std::addressof(__c)) {
     __base::__copy_assign_alloc(__c);
     assign(__c.begin(), __c.end());
@@ -1099,13 +1179,14 @@ inline list<_Tp, _Alloc>& list<_Tp, _Alloc>::operator=(const list& __c) {
 
 template <class _Tp, class _Alloc>
 template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
-void list<_Tp, _Alloc>::assign(_InpIter __f, _InpIter __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::assign(_InpIter __f, _InpIter __l) {
   __assign_with_sentinel(__f, __l);
 }
 
 template <class _Tp, class _Alloc>
 template <class _Iterator, class _Sentinel>
-_LIBCPP_HIDE_FROM_ABI void list<_Tp, _Alloc>::__assign_with_sentinel(_Iterator __f, _Sentinel __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI void
+list<_Tp, _Alloc>::__assign_with_sentinel(_Iterator __f, _Sentinel __l) {
   iterator __i = begin();
   iterator __e = end();
   for (; __f != __l && __i != __e; ++__f, (void)++__i)
@@ -1117,7 +1198,7 @@ _LIBCPP_HIDE_FROM_ABI void list<_Tp, _Alloc>::__assign_with_sentinel(_Iterator _
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::assign(size_type __n, const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::assign(size_type __n, const value_type& __x) {
   iterator __i = begin();
   iterator __e = end();
   for (; __n > 0 && __i != __e; --__n, (void)++__i)
@@ -1129,12 +1210,13 @@ void list<_Tp, _Alloc>::assign(size_type __n, const value_type& __x) {
 }
 
 template <class _Tp, class _Alloc>
-inline _Alloc list<_Tp, _Alloc>::get_allocator() const _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _Alloc list<_Tp, _Alloc>::get_allocator() const _NOEXCEPT {
   return allocator_type(this->__node_alloc_);
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::insert(const_iterator __p, const value_type& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
   __link_nodes(__p.__ptr_, __node->__as_link(), __node->__as_link());
   ++this->__size_;
@@ -1142,7 +1224,7 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
 list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& __x) {
   iterator __r(__p.__ptr_);
   if (__n > 0) {
@@ -1178,13 +1260,14 @@ list<_Tp, _Alloc>::insert(const_iterator __p, size_type __n, const value_type& _
 
 template <class _Tp, class _Alloc>
 template <class _InpIter, __enable_if_t<__has_input_iterator_category<_InpIter>::value, int> >
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, _InpIter __f, _InpIter __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::insert(const_iterator __p, _InpIter __f, _InpIter __l) {
   return __insert_with_sentinel(__p, __f, __l);
 }
 
 template <class _Tp, class _Alloc>
 template <class _Iterator, class _Sentinel>
-_LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Alloc>::iterator
 list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Sentinel __l) {
   iterator __r(__p.__ptr_);
   if (__f != __l) {
@@ -1219,7 +1302,7 @@ list<_Tp, _Alloc>::__insert_with_sentinel(const_iterator __p, _Iterator __f, _Se
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::push_front(const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::push_front(const value_type& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
   __base_pointer __nl   = __node->__as_link();
   __link_nodes_at_front(__nl, __nl);
@@ -1227,7 +1310,7 @@ void list<_Tp, _Alloc>::push_front(const value_type& __x) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::push_back(const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::push_back(const value_type& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, __x);
   __base_pointer __nl   = __node->__as_link();
   __link_nodes_at_back(__nl, __nl);
@@ -1237,7 +1320,7 @@ void list<_Tp, _Alloc>::push_back(const value_type& __x) {
 #  ifndef _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::push_front(value_type&& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::push_front(value_type&& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
   __base_pointer __nl   = __node->__as_link();
   __link_nodes_at_front(__nl, __nl);
@@ -1245,7 +1328,7 @@ void list<_Tp, _Alloc>::push_front(value_type&& __x) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::push_back(value_type&& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::push_back(value_type&& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
   __base_pointer __nl   = __node->__as_link();
   __link_nodes_at_back(__nl, __nl);
@@ -1254,12 +1337,13 @@ void list<_Tp, _Alloc>::push_back(value_type&& __x) {
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 #    if _LIBCPP_STD_VER >= 17
-typename list<_Tp, _Alloc>::reference
+    typename list<_Tp, _Alloc>::reference
 #    else
-void
+    void
 #    endif
-list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
+    list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
   __node_pointer __node =
       this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
   __base_pointer __nl = __node->__as_link();
@@ -1272,12 +1356,13 @@ list<_Tp, _Alloc>::emplace_front(_Args&&... __args) {
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
+_LIBCPP_CONSTEXPR_SINCE_CXX26
 #    if _LIBCPP_STD_VER >= 17
-typename list<_Tp, _Alloc>::reference
+    typename list<_Tp, _Alloc>::reference
 #    else
-void
+    void
 #    endif
-list<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
+    list<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
   __node_pointer __node =
       this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
   __base_pointer __nl = __node->__as_link();
@@ -1290,7 +1375,8 @@ list<_Tp, _Alloc>::emplace_back(_Args&&... __args) {
 
 template <class _Tp, class _Alloc>
 template <class... _Args>
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::emplace(const_iterator __p, _Args&&... __args) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::emplace(const_iterator __p, _Args&&... __args) {
   __node_pointer __node =
       this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::forward<_Args>(__args)...);
   __base_pointer __nl = __node->__as_link();
@@ -1300,7 +1386,8 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::emplace(const_iterator _
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __p, value_type&& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::insert(const_iterator __p, value_type&& __x) {
   __node_pointer __node = this->__create_node(/* prev = */ nullptr, /* next = */ nullptr, std::move(__x));
   __base_pointer __nl   = __node->__as_link();
   __link_nodes(__p.__ptr_, __nl, __nl);
@@ -1311,7 +1398,7 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::insert(const_iterator __
 #  endif // _LIBCPP_CXX03_LANG
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::pop_front() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::pop_front() {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::pop_front() called with empty list");
   __base_pointer __n = __base::__end_.__next_;
   __base::__unlink_nodes(__n, __n);
@@ -1320,7 +1407,7 @@ void list<_Tp, _Alloc>::pop_front() {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::pop_back() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::pop_back() {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(!empty(), "list::pop_back() called on an empty list");
   __base_pointer __n = __base::__end_.__prev_;
   __base::__unlink_nodes(__n, __n);
@@ -1329,7 +1416,7 @@ void list<_Tp, _Alloc>::pop_back() {
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __p) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __p) {
   _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(__p != end(), "list::erase(iterator) called with a non-dereferenceable iterator");
   __base_pointer __n = __p.__ptr_;
   __base_pointer __r = __n->__next_;
@@ -1340,7 +1427,8 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __p
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __f, const_iterator __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
+list<_Tp, _Alloc>::erase(const_iterator __f, const_iterator __l) {
   if (__f != __l) {
     __base::__unlink_nodes(__f.__ptr_, __l.__ptr_->__prev_);
     while (__f != __l) {
@@ -1354,7 +1442,7 @@ typename list<_Tp, _Alloc>::iterator list<_Tp, _Alloc>::erase(const_iterator __f
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::resize(size_type __n) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n) {
   if (__n < this->__size_)
     erase(__iterator(__n), end());
   else if (__n > this->__size_) {
@@ -1389,7 +1477,7 @@ void list<_Tp, _Alloc>::resize(size_type __n) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
   if (__n < this->__size_)
     erase(__iterator(__n), end());
   else if (__n > this->__size_) {
@@ -1425,7 +1513,7 @@ void list<_Tp, _Alloc>::resize(size_type __n, const value_type& __x) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c) {
   _LIBCPP_ASSERT_VALID_INPUT_RANGE(
       this != std::addressof(__c), "list::splice(iterator, list) called with this == &list");
   if (!__c.empty()) {
@@ -1439,7 +1527,7 @@ void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c) {
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __i) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __i) {
   if (__p.__ptr_ != __i.__ptr_ && __p.__ptr_ != __i.__ptr_->__next_) {
     __base_pointer __f = __i.__ptr_;
     __base::__unlink_nodes(__f, __f);
@@ -1450,7 +1538,8 @@ void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __i
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void
+list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __f, const_iterator __l) {
   if (__f != __l) {
     __base_pointer __first = __f.__ptr_;
     --__l;
@@ -1466,7 +1555,8 @@ void list<_Tp, _Alloc>::splice(const_iterator __p, list& __c, const_iterator __f
 }
 
 template <class _Tp, class _Alloc>
-typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove(const value_type& __x) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::__remove_return_type
+list<_Tp, _Alloc>::remove(const value_type& __x) {
   list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   for (const_iterator __i = begin(), __e = end(); __i != __e;) {
     if (*__i == __x) {
@@ -1486,7 +1576,8 @@ typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove(const
 
 template <class _Tp, class _Alloc>
 template <class _Pred>
-typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove_if(_Pred __pred) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::__remove_return_type
+list<_Tp, _Alloc>::remove_if(_Pred __pred) {
   list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   for (iterator __i = begin(), __e = end(); __i != __e;) {
     if (__pred(*__i)) {
@@ -1506,7 +1597,8 @@ typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::remove_if(_P
 
 template <class _Tp, class _Alloc>
 template <class _BinaryPred>
-typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::unique(_BinaryPred __binary_pred) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::__remove_return_type
+list<_Tp, _Alloc>::unique(_BinaryPred __binary_pred) {
   list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   for (iterator __i = begin(), __e = end(); __i != __e;) {
     iterator __j = std::next(__i);
@@ -1522,13 +1614,13 @@ typename list<_Tp, _Alloc>::__remove_return_type list<_Tp, _Alloc>::unique(_Bina
 }
 
 template <class _Tp, class _Alloc>
-inline void list<_Tp, _Alloc>::merge(list& __c) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void list<_Tp, _Alloc>::merge(list& __c) {
   merge(__c, __less<>());
 }
 
 template <class _Tp, class _Alloc>
 template <class _Comp>
-void list<_Tp, _Alloc>::merge(list& __c, _Comp __comp) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::merge(list& __c, _Comp __comp) {
   if (this != std::addressof(__c)) {
     iterator __f1 = begin();
     iterator __e1 = end();
@@ -1557,19 +1649,19 @@ void list<_Tp, _Alloc>::merge(list& __c, _Comp __comp) {
 }
 
 template <class _Tp, class _Alloc>
-inline void list<_Tp, _Alloc>::sort() {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void list<_Tp, _Alloc>::sort() {
   sort(__less<>());
 }
 
 template <class _Tp, class _Alloc>
 template <class _Comp>
-inline void list<_Tp, _Alloc>::sort(_Comp __comp) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline void list<_Tp, _Alloc>::sort(_Comp __comp) {
   __sort(begin(), end(), this->__size_, __comp);
 }
 
 template <class _Tp, class _Alloc>
 template <class _Comp>
-typename list<_Tp, _Alloc>::iterator
+_LIBCPP_CONSTEXPR_SINCE_CXX26 typename list<_Tp, _Alloc>::iterator
 list<_Tp, _Alloc>::__sort(iterator __f1, iterator __e2, size_type __n, _Comp& __comp) {
   switch (__n) {
   case 0:
@@ -1623,7 +1715,7 @@ list<_Tp, _Alloc>::__sort(iterator __f1, iterator __e2, size_type __n, _Comp& __
 }
 
 template <class _Tp, class _Alloc>
-void list<_Tp, _Alloc>::reverse() _NOEXCEPT {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 void list<_Tp, _Alloc>::reverse() _NOEXCEPT {
   if (this->__size_ > 1) {
     iterator __e = end();
     for (iterator __i = begin(); __i.__ptr_ != __e.__ptr_;) {
@@ -1635,46 +1727,52 @@ void list<_Tp, _Alloc>::reverse() _NOEXCEPT {
 }
 
 template <class _Tp, class _Alloc>
-bool list<_Tp, _Alloc>::__invariants() const {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 bool list<_Tp, _Alloc>::__invariants() const {
   return size() == std::distance(begin(), end());
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator==(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator==(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return __x.size() == __y.size() && std::equal(__x.begin(), __x.end(), __y.begin());
 }
 
 #  if _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator<(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator<(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return std::lexicographical_compare(__x.begin(), __x.end(), __y.begin(), __y.end());
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator!=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator!=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return !(__x == __y);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator>(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator>(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return __y < __x;
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator>=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator>=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return !(__x < __y);
 }
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI bool operator<=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI bool
+operator<=(const list<_Tp, _Alloc>& __x, const list<_Tp, _Alloc>& __y) {
   return !(__y < __x);
 }
 
 #  else // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Allocator>
-_LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
+_LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __synth_three_way_result<_Tp>
 operator<=>(const list<_Tp, _Allocator>& __x, const list<_Tp, _Allocator>& __y) {
   return std::lexicographical_compare_three_way(__x.begin(), __x.end(), __y.begin(), __y.end(), std::__synth_three_way);
 }
@@ -1682,20 +1780,20 @@ operator<=>(const list<_Tp, _Allocator>& __x, const list<_Tp, _Allocator>& __y)
 #  endif // _LIBCPP_STD_VER <= 17
 
 template <class _Tp, class _Alloc>
-inline _LIBCPP_HIDE_FROM_ABI void swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>& __y)
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI void swap(list<_Tp, _Alloc>& __x, list<_Tp, _Alloc>& __y)
     _NOEXCEPT_(_NOEXCEPT_(__x.swap(__y))) {
   __x.swap(__y);
 }
 
 #  if _LIBCPP_STD_VER >= 20
 template <class _Tp, class _Allocator, class _Predicate>
-inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
 erase_if(list<_Tp, _Allocator>& __c, _Predicate __pred) {
   return __c.remove_if(__pred);
 }
 
 template <class _Tp, class _Allocator, class _Up>
-inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
+_LIBCPP_CONSTEXPR_SINCE_CXX26 inline _LIBCPP_HIDE_FROM_ABI typename list<_Tp, _Allocator>::size_type
 erase(list<_Tp, _Allocator>& __c, const _Up& __v) {
   return std::erase_if(__c, [&](const auto& __elem) -> bool { return __elem == __v; });
 }
diff --git a/libcxx/include/version b/libcxx/include/version
index 87c4ede9a7e5..7154cab92335 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -71,6 +71,7 @@ __cpp_lib_constexpr_dynamic_alloc                       201907L <memory>
 __cpp_lib_constexpr_forward_list                        202502L <forward_list>
 __cpp_lib_constexpr_functional                          201907L <functional>
 __cpp_lib_constexpr_iterator                            201811L <iterator>
+__cpp_lib_constexpr_list                                202502L <list>
 __cpp_lib_constexpr_memory                              202202L <memory>
                                                         201811L // C++20
 __cpp_lib_constexpr_new                                 202406L <new>
@@ -545,6 +546,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # undef  __cpp_lib_constexpr_algorithms
 # define __cpp_lib_constexpr_algorithms                 202306L
 # define __cpp_lib_constexpr_forward_list               202502L
+# define __cpp_lib_constexpr_list                       202502L
 # if !defined(_LIBCPP_ABI_VCRUNTIME)
 #   define __cpp_lib_constexpr_new                      202406L
 # endif
diff --git a/libcxx/test/std/containers/sequences/list/compare.pass.cpp b/libcxx/test/std/containers/sequences/list/compare.pass.cpp
index ce00f57733bf..9705fd916198 100644
--- a/libcxx/test/std/containers/sequences/list/compare.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/compare.pass.cpp
@@ -10,34 +10,34 @@
 
 // template< class T, class Alloc >
 // bool operator==( const std::list<T,Alloc>& lhs,
-//                  const std::list<T,Alloc>& rhs );
+//                  const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 // template< class T, class Alloc >
 // bool operator!=( const std::list<T,Alloc>& lhs,
-//                  const std::list<T,Alloc>& rhs );
+//                  const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 // template< class T, class Alloc >
 // bool operator<( const std::list<T,Alloc>& lhs,
-//                 const std::list<T,Alloc>& rhs );
+//                 const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 // template< class T, class Alloc >
 // bool operator<=( const std::list<T,Alloc>& lhs,
-//                  const std::list<T,Alloc>& rhs );
+//                  const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 // template< class T, class Alloc >
 // bool operator>( const std::list<T,Alloc>& lhs,
-//                 const std::list<T,Alloc>& rhs );
+//                 const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 // template< class T, class Alloc >
 // bool operator>=( const std::list<T,Alloc>& lhs,
-//                  const std::list<T,Alloc>& rhs );
+//                  const std::list<T,Alloc>& rhs ); // constexpr since C++26
 
 #include <list>
 #include <cassert>
 
 #include "test_comparisons.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     const std::list<int> l1, l2;
     assert(testComparisons(l1, l2, true, false));
@@ -113,5 +113,15 @@ int main(int, char**) {
     const std::list<LessAndEqComp> l2(items2, items2 + 2);
     assert(testComparisons(l1, l2, false, false));
   }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/compare.three_way.pass.cpp b/libcxx/test/std/containers/sequences/list/compare.three_way.pass.cpp
index 059fba3c2626..7a23a653c0aa 100644
--- a/libcxx/test/std/containers/sequences/list/compare.three_way.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/compare.three_way.pass.cpp
@@ -11,7 +11,7 @@
 
 // template <class T, class Allocator> constexpr
 //   synth-three-way-result<T>
-//     operator<=>(const list<T, Allocator>& x, const list<T, Allocator>& y);
+//     operator<=>(const list<T, Allocator>& x, const list<T, Allocator>& y); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -20,6 +20,8 @@
 
 int main(int, char**) {
   assert(test_sequence_container_spaceship<std::list>());
-  // `std::list` is not constexpr, so no `static_assert` test here.
+#if TEST_STD_VER >= 26
+  static_assert(test_sequence_container_spaceship<std::list>());
+#endif
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp b/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
index f1002f2ca811..9d724673d31e 100644
--- a/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/get_allocator.pass.cpp
@@ -10,7 +10,7 @@
 
 // class list
 
-// allocator_type get_allocator() const
+// allocator_type get_allocator() const // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::allocator<int> alloc;
     const std::list<int> l(alloc);
@@ -30,5 +30,14 @@ int main(int, char**) {
     assert(l.get_allocator() == alloc);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp b/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
index 1802e53ecf38..ac8a76097d0a 100644
--- a/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/incomplete_type.pass.cpp
@@ -12,6 +12,7 @@
 // type.
 
 #include <list>
+#include <cassert>
 
 #include "test_macros.h"
 
@@ -23,8 +24,18 @@ struct A {
   std::list<A>::const_reverse_iterator crit;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   A a;
+  (void)a;
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/iterators.pass.cpp b/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
index deaae31f2d27..b41a1899f2ff 100644
--- a/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/iterators.pass.cpp
@@ -8,12 +8,12 @@
 
 // <list>
 
-// iterator       begin();
-// iterator       end();
-// const_iterator begin()  const;
-// const_iterator end()    const;
-// const_iterator cbegin() const;
-// const_iterator cend()   const;
+// iterator       begin();        // constexpr since C++26
+// iterator       end();          // constexpr since C++26
+// const_iterator begin()  const; // constexpr since C++26
+// const_iterator end()    const; // constexpr since C++26
+// const_iterator cbegin() const; // constexpr since C++26
+// const_iterator cend()   const; // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -27,7 +27,7 @@ struct A {
   int second;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef int T;
     typedef std::list<T> C;
@@ -74,6 +74,8 @@ int main(int, char**) {
     typedef std::list<T> C;
     C::iterator i;
     C::const_iterator j;
+    (void)i;
+    (void)j;
   }
 #if TEST_STD_VER >= 11
   {
@@ -122,6 +124,8 @@ int main(int, char**) {
     typedef std::list<T, min_allocator<T>> C;
     C::iterator i;
     C::const_iterator j;
+    (void)i;
+    (void)j;
   }
   {
     typedef A T;
@@ -150,5 +154,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
index 50ca23ff9c56..f368d8e700bb 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/empty.pass.cpp
@@ -10,7 +10,7 @@
 
 // class list
 
-// bool empty() const noexcept;
+// bool empty() const noexcept; // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef std::list<int> C;
     C c;
@@ -42,5 +42,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
index 74c2ccfb1442..1f956b33e482 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/max_size.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// size_type max_size() const noexcept
+// size_type max_size() const noexcept // constexpr since C++26
 
 #include <cassert>
 #include <limits>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef limited_allocator<int, 10> A;
     typedef std::list<int, A> C;
@@ -42,5 +42,14 @@ int main(int, char**) {
     assert(c.max_size() <= alloc_max_size(c.get_allocator()));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
index 754d931646cc..f694d9ab2650 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size.pass.cpp
@@ -8,15 +8,16 @@
 
 // <list>
 
-// void resize(size_type sz);
+// void resize(size_type sz); // constexpr since C++26
 
 #include <list>
 #include <cassert>
+
 #include "test_macros.h"
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l(5, 2);
     l.resize(2);
@@ -33,17 +34,31 @@ int main(int, char**) {
     assert(l.back() == 0);
   }
 #if TEST_STD_VER >= 11
-  {
-    std::list<DefaultOnly> l(10);
-    l.resize(5);
-    assert(l.size() == 5);
-    assert(std::distance(l.begin(), l.end()) == 5);
-  }
-  {
-    std::list<DefaultOnly> l(10);
-    l.resize(20);
-    assert(l.size() == 20);
-    assert(std::distance(l.begin(), l.end()) == 20);
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    {
+      std::list<DefaultOnly> l(10);
+      l.resize(5);
+      assert(l.size() == 5);
+      assert(std::distance(l.begin(), l.end()) == 5);
+    }
+    {
+      std::list<DefaultOnly> l(10);
+      l.resize(20);
+      assert(l.size() == 20);
+      assert(std::distance(l.begin(), l.end()) == 20);
+    }
+    {
+      std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
+      l.resize(5);
+      assert(l.size() == 5);
+      assert(std::distance(l.begin(), l.end()) == 5);
+    }
+    {
+      std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
+      l.resize(20);
+      assert(l.size() == 20);
+      assert(std::distance(l.begin(), l.end()) == 20);
+    }
   }
   {
     std::list<int, min_allocator<int>> l(5, 2);
@@ -60,18 +75,15 @@ int main(int, char**) {
     assert(l.front() == 2);
     assert(l.back() == 0);
   }
-  {
-    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
-    l.resize(5);
-    assert(l.size() == 5);
-    assert(std::distance(l.begin(), l.end()) == 5);
-  }
-  {
-    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(10);
-    l.resize(20);
-    assert(l.size() == 20);
-    assert(std::distance(l.begin(), l.end()) == 20);
-  }
+#endif
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
 #endif
 
   return 0;
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
index 95fccddeca54..a93ec224bd6d 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/resize_size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void resize(size_type sz, const value_type& x);
+// void resize(size_type sz, const value_type& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<double> l(5, 2);
     l.resize(2, 3.5);
@@ -50,5 +50,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp b/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
index 930331205a9a..8aecfcaea027 100644
--- a/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.capacity/size.pass.cpp
@@ -10,7 +10,7 @@
 
 // class list
 
-// size_type size() const noexcept;
+// size_type size() const noexcept; // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef std::list<int> C;
     C c;
@@ -58,5 +58,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
index ca468d870998..912975d55e1d 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_copy.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// list& operator=(const list& c);
+// list& operator=(const list& c); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
     std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
@@ -41,5 +41,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
index d4c1120df622..07b25f189a11 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_initializer_list.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// void assign(initializer_list<value_type> il);
+// void assign(initializer_list<value_type> il); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> d;
     d.assign({3, 4, 5, 6});
@@ -40,5 +40,14 @@ int main(int, char**) {
     assert(*i++ == 6);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
index 87faaaac2b21..aa199b05ed45 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/assign_move.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// list& operator=(list&& c);
+// list& operator=(list&& c); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
     std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
@@ -79,5 +79,14 @@ int main(int, char**) {
     assert(it == l2.begin()); // Iterators remain valid
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
index de52da0fefab..a3e510d4d6eb 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/copy.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// list(const list& c);
+// list(const list& c); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l(3, 2);
     std::list<int> l2 = l;
@@ -50,5 +50,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  test();
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
index 0d6c6f431f09..5da17a9c9b59 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/copy_alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// list(const list& c, const allocator_type& a);
+// list(const list& c, const allocator_type& a); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int, test_allocator<int> > l(3, 2, test_allocator<int>(5));
     std::list<int, test_allocator<int> > l2(l, test_allocator<int>(3));
@@ -39,5 +39,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
index 0cfd8f1e9c59..1256433659c6 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/default.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// explicit list(const Alloc& = Alloc());
+// explicit list(const Alloc& = Alloc()); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "DefaultOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l;
     assert(l.size() == 0);
@@ -65,5 +65,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
index e31a58d7b9a5..3a78d0e0e0d5 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/default_stack_alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// explicit list(const Alloc& = Alloc());
+// explicit list(const Alloc& = Alloc()); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l;
     assert(l.size() == 0);
@@ -45,5 +45,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
index cc5ed5729b57..311c72d815d1 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/from_range.pass.cpp
@@ -9,14 +9,15 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23
+//   list(from_range_t, R&& rg, const Allocator& = Allocator()); // C++23; constexpr since C++26
 
 #include <list>
+#include <type_traits>
 
 #include "../../from_range_sequence_containers.h"
 #include "test_macros.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   for_all_iterators_and_allocators<int>([]<class Iter, class Sent, class Alloc>() {
     test_sequence_container<std::list, int, Iter, Sent, Alloc>([](const auto&) {
       // No additional validation to do.
@@ -26,8 +27,19 @@ int main(int, char**) {
 
   static_assert(test_constraints<std::list, int, double>());
 
-  test_exception_safety_throwing_copy<std::list>();
-  test_exception_safety_throwing_allocator<std::list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_exception_safety_throwing_copy<std::list>();
+    test_exception_safety_throwing_allocator<std::list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
index 3ba90d1337e9..9e3a71ed3bd1 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// list(initializer_list<value_type> il);
+// list(initializer_list<value_type> il); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> d = {3, 4, 5, 6};
     assert(d.size() == 4);
@@ -38,5 +38,14 @@ int main(int, char**) {
     assert(*i++ == 6);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
index e4779eb5a640..1b6b1e19c6eb 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/initializer_list_alloc.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// list(initializer_list<value_type> il, const Allocator& a = allocator_type());
+// list(initializer_list<value_type> il, const Allocator& a = allocator_type()); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int, test_allocator<int>> d({3, 4, 5, 6}, test_allocator<int>(3));
     assert(d.get_allocator() == test_allocator<int>(3));
@@ -41,5 +41,14 @@ int main(int, char**) {
     assert(*i++ == 6);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
index c99069f92f51..d92307283098 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/input_iterator.pass.cpp
@@ -9,10 +9,11 @@
 // <list>
 
 // template <class InputIterator>
-//   list(InputIterator first, InputIterator last, const Allocator& = Allocator());
+//   list(InputIterator first, InputIterator last, const Allocator& = Allocator()); // constexpr since C++26
 
 #include <list>
 #include <cassert>
+
 #include "test_macros.h"
 #include "test_iterators.h"
 #include "test_allocator.h"
@@ -22,7 +23,7 @@
 #  include "container_test_types.h"
 #endif
 
-void basic_test() {
+TEST_CONSTEXPR_CXX26 void basic_test() {
   {
     int a[] = {0, 1, 2, 3};
     std::list<int> l(
@@ -81,7 +82,7 @@ void basic_test() {
 #endif
 }
 
-void test_emplacable_concept() {
+TEST_CONSTEXPR_CXX26 void test_emplacable_concept() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
@@ -126,7 +127,7 @@ void test_emplacable_concept() {
 #endif
 }
 
-void test_emplacable_concept_with_alloc() {
+TEST_CONSTEXPR_CXX26 void test_emplacable_concept_with_alloc() {
 #if TEST_STD_VER >= 11
   int arr1[] = {42};
   int arr2[] = {1, 101, 42};
@@ -239,12 +240,24 @@ void test_ctor_under_alloc_with_alloc() {
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   basic_test();
   test_emplacable_concept();
   test_emplacable_concept_with_alloc();
-  test_ctor_under_alloc();
-  test_ctor_under_alloc_with_alloc();
+
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_ctor_under_alloc();
+    test_ctor_under_alloc_with_alloc();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
index 6703390f10b9..cae2886cf08b 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// list(list&& c);
+// list(list&& c); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
     std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
@@ -63,5 +63,14 @@ int main(int, char**) {
     assert(it == l2.begin()); // Iterators remain valid
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
index f6a1f2c33a63..dee0282c9978 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/move_alloc.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// list(list&& c, const allocator_type& a);
+// list(list&& c, const allocator_type& a); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly, test_allocator<MoveOnly> > l(test_allocator<MoveOnly>(5));
     std::list<MoveOnly, test_allocator<MoveOnly> > lo(test_allocator<MoveOnly>(5));
@@ -69,5 +69,14 @@ int main(int, char**) {
     assert(l2.get_allocator() == min_allocator<MoveOnly>());
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
index a9ab30b82640..d7679931ee71 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/op_equal_initializer_list.pass.cpp
@@ -10,14 +10,14 @@
 
 // <list>
 
-// list& operator=(initializer_list<value_type> il);
+// list& operator=(initializer_list<value_type> il); // constexpr since C++26
 
 #include <list>
 #include <cassert>
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> d;
     d = {3, 4, 5, 6};
@@ -39,5 +39,14 @@ int main(int, char**) {
     assert(*i++ == 6);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
index 937a86a27e05..55371e8354a9 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/size_type.pass.cpp
@@ -8,18 +8,19 @@
 
 // <list>
 
-// explicit list(size_type n);
+// explicit list(size_type n); // constexpr since C++26
 
 #include <list>
 #include <cassert>
 #include <cstddef>
+
 #include "test_macros.h"
 #include "DefaultOnly.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
 
 template <class T, class Allocator>
-void test3(unsigned n, Allocator const& alloc = Allocator()) {
+TEST_CONSTEXPR_CXX26 void test1(unsigned n, Allocator const& alloc = Allocator()) {
 #if TEST_STD_VER > 11
   typedef std::list<T, Allocator> C;
   {
@@ -34,7 +35,7 @@ void test3(unsigned n, Allocator const& alloc = Allocator()) {
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l(3);
     assert(l.size() == 3);
@@ -70,15 +71,10 @@ int main(int, char**) {
     assert(*i == 0);
     ++i;
     assert(*i == 0);
-    test3<int, min_allocator<int>>(3);
+    test1<int, min_allocator<int>>(3);
   }
 #endif
 #if TEST_STD_VER >= 11
-  {
-    std::list<DefaultOnly> l(3);
-    assert(l.size() == 3);
-    assert(std::distance(l.begin(), l.end()) == 3);
-  }
   {
     std::list<int, min_allocator<int>> l(3);
     assert(l.size() == 3);
@@ -90,12 +86,29 @@ int main(int, char**) {
     ++i;
     assert(*i == 0);
   }
-  {
-    std::list<DefaultOnly, min_allocator<DefaultOnly>> l(3);
-    assert(l.size() == 3);
-    assert(std::distance(l.begin(), l.end()) == 3);
+
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    {
+      std::list<DefaultOnly> l(3);
+      assert(l.size() == 3);
+      assert(std::distance(l.begin(), l.end()) == 3);
+    }
+    {
+      std::list<DefaultOnly, min_allocator<DefaultOnly>> l(3);
+      assert(l.size() == 3);
+      assert(std::distance(l.begin(), l.end()) == 3);
+    }
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp b/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
index ff7982ce147d..42700c3ed658 100644
--- a/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.cons/size_value_alloc.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// list(size_type n, const T& value, const Allocator& = Allocator());
+// list(size_type n, const T& value, const Allocator& = Allocator()); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> l(3, 2);
     assert(l.size() == 3);
@@ -77,5 +77,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp b/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
index 77f9f8956037..babd4b2758e6 100644
--- a/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.erasure/erase.pass.cpp
@@ -11,7 +11,7 @@
 
 // template <class T, class Allocator, class U>
 //   typename list<T, Allocator>::size_type
-//   erase(list<T, Allocator>& c, const U& value);
+//   erase(list<T, Allocator>& c, const U& value); // constexpr since C++26
 
 #include <list>
 #include <optional>
@@ -21,14 +21,14 @@
 #include "min_allocator.h"
 
 template <class S, class U>
-void test0(S s, U val, S expected, std::size_t expected_erased_count) {
+TEST_CONSTEXPR_CXX26 void test0(S s, U val, S expected, std::size_t expected_erased_count) {
   ASSERT_SAME_TYPE(typename S::size_type, decltype(std::erase(s, val)));
   assert(expected_erased_count == std::erase(s, val));
   assert(s == expected);
 }
 
 template <class S>
-void test() {
+TEST_CONSTEXPR_CXX26 void test1() {
   test0(S(), 1, S(), 0);
 
   test0(S({1}), 1, S(), 1);
@@ -62,13 +62,22 @@ void test() {
   test0(S({1, 2, 1}), opt(3), S({1, 2, 1}), 0);
 }
 
-int main(int, char**) {
-  test<std::list<int>>();
-  test<std::list<int, min_allocator<int>>>();
-  test<std::list<int, test_allocator<int>>>();
+TEST_CONSTEXPR_CXX26 bool test() {
+  test1<std::list<int>>();
+  test1<std::list<int, min_allocator<int>>>();
+  test1<std::list<int, test_allocator<int>>>();
+
+  test1<std::list<long>>();
+  test1<std::list<double>>();
 
-  test<std::list<long>>();
-  test<std::list<double>>();
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp b/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
index 5352a2f454f8..e396330bc68c 100644
--- a/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.erasure/erase_if.pass.cpp
@@ -11,7 +11,7 @@
 
 // template <class T, class Allocator, class Predicate>
 //   typename list<T, Allocator>::size_type
-//   erase_if(list<T, Allocator>& c, Predicate pred);
+//   erase_if(list<T, Allocator>& c, Predicate pred); // constexpr since C++26
 
 #include <list>
 
@@ -20,14 +20,14 @@
 #include "min_allocator.h"
 
 template <class S, class Pred>
-void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
+TEST_CONSTEXPR_CXX26 void test0(S s, Pred p, S expected, std::size_t expected_erased_count) {
   ASSERT_SAME_TYPE(typename S::size_type, decltype(std::erase_if(s, p)));
   assert(expected_erased_count == std::erase_if(s, p));
   assert(s == expected);
 }
 
 template <typename S>
-void test() {
+TEST_CONSTEXPR_CXX26 void test1() {
   auto is1   = [](auto v) { return v == 1; };
   auto is2   = [](auto v) { return v == 2; };
   auto is3   = [](auto v) { return v == 3; };
@@ -64,13 +64,22 @@ void test() {
   test0(S({1, 2, 3}), False, S({1, 2, 3}), 0);
 }
 
-int main(int, char**) {
-  test<std::list<int>>();
-  test<std::list<int, min_allocator<int>>>();
-  test<std::list<int, test_allocator<int>>>();
+TEST_CONSTEXPR_CXX26 bool test() {
+  test1<std::list<int>>();
+  test1<std::list<int, min_allocator<int>>>();
+  test1<std::list<int, test_allocator<int>>>();
+
+  test1<std::list<long>>();
+  test1<std::list<double>>();
 
-  test<std::list<long>>();
-  test<std::list<double>>();
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/append_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/append_range.pass.cpp
index 46a99cb54844..4b47a8738e52 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/append_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/append_range.pass.cpp
@@ -9,9 +9,10 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr void append_range(R&& rg); // C++23
+//   constexpr void append_range(R&& rg); // C++23; constexpr since C++26
 
 #include <list>
+#include <type_traits>
 
 #include "../../insert_range_sequence_containers.h"
 #include "test_macros.h"
@@ -21,7 +22,7 @@
 //   {empty/one-element/full} container);
 // - appending move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_append_range<std::list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +32,19 @@ int main(int, char**) {
   });
   test_sequence_append_range_move_only<std::list>();
 
-  test_append_range_exception_safety_throwing_copy<std::list>();
-  test_append_range_exception_safety_throwing_allocator<std::list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_append_range_exception_safety_throwing_copy<std::list>();
+    test_append_range_exception_safety_throwing_allocator<std::list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/assign_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/assign_range.pass.cpp
index d745786b6815..83a12879a041 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/assign_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/assign_range.pass.cpp
@@ -9,9 +9,10 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr void assign_range(R&& rg); // C++23
+//   constexpr void assign_range(R&& rg); // C++23; constexpr since C++26
 
 #include <list>
+#include <type_traits>
 
 #include "../../insert_range_sequence_containers.h"
 #include "test_macros.h"
@@ -21,7 +22,7 @@
 //   {empty/one-element/full} container);
 // - assigning move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_assign_range<std::list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +32,19 @@ int main(int, char**) {
   });
   test_sequence_assign_range_move_only<std::list>();
 
-  test_assign_range_exception_safety_throwing_copy<std::list>();
-  test_assign_range_exception_safety_throwing_allocator<std::list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_assign_range_exception_safety_throwing_copy<std::list>();
+    test_assign_range_exception_safety_throwing_allocator<std::list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
index 5931fd62d037..0b38ae05bd68 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/clear.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void clear() noexcept;
+// void clear() noexcept; // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a[] = {1, 2, 3};
     std::list<int> c(a, a + 3);
@@ -34,5 +34,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
index 2f83aa0d317b..9bd7a151d20e 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// template <class... Args> void emplace(const_iterator p, Args&&... args);
+// template <class... Args> void emplace(const_iterator p, Args&&... args); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -26,13 +26,13 @@ class A {
   A& operator=(const A&);
 
 public:
-  A(int i, double d) : i_(i), d_(d) {}
+  TEST_CONSTEXPR_CXX20 A(int i, double d) : i_(i), d_(d) {}
 
-  int geti() const { return i_; }
-  double getd() const { return d_; }
+  TEST_CONSTEXPR int geti() const { return i_; }
+  TEST_CONSTEXPR double getd() const { return d_; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<A> c;
     c.emplace(c.cbegin(), 2, 3.5);
@@ -60,5 +60,14 @@ int main(int, char**) {
     assert(c.back().getd() == 4.5);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
index 900f8b83d3e6..5f84c4c7c05a 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_back.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// template <class... Args> reference emplace_back(Args&&... args);
+// template <class... Args> reference emplace_back(Args&&... args); // constexpr since C++26
 // return type is 'reference' in C++17; 'void' before
 
 #include <list>
@@ -27,13 +27,13 @@ class A {
   A& operator=(const A&);
 
 public:
-  A(int i, double d) : i_(i), d_(d) {}
+  TEST_CONSTEXPR_CXX20 A(int i, double d) : i_(i), d_(d) {}
 
-  int geti() const { return i_; }
-  double getd() const { return d_; }
+  TEST_CONSTEXPR int geti() const { return i_; }
+  TEST_CONSTEXPR double getd() const { return d_; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<A> c;
 #if TEST_STD_VER > 14
@@ -83,5 +83,14 @@ int main(int, char**) {
     assert(c.back().getd() == 4.5);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
index 665f5077bd42..95474b52dbd0 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/emplace_front.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// template <class... Args> reference emplace_front(Args&&... args);
+// template <class... Args> reference emplace_front(Args&&... args); // constexpr since C++26
 // return type is 'reference' in C++17; 'void' before
 
 #include <list>
@@ -27,13 +27,13 @@ class A {
   A& operator=(const A&);
 
 public:
-  A(int i, double d) : i_(i), d_(d) {}
+  TEST_CONSTEXPR_CXX20 A(int i, double d) : i_(i), d_(d) {}
 
-  int geti() const { return i_; }
-  double getd() const { return d_; }
+  TEST_CONSTEXPR int geti() const { return i_; }
+  TEST_CONSTEXPR double getd() const { return d_; }
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<A> c;
 #if TEST_STD_VER > 14
@@ -84,5 +84,14 @@ int main(int, char**) {
     assert(c.back().getd() == 3.5);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
index ba139b4367d7..79dae11a8263 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// iterator erase(const_iterator position);
+// iterator erase(const_iterator position); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {1, 2, 3};
     std::list<int> l1(a1, a1 + 3);
@@ -62,5 +62,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
index cc8d537032d0..fa6f8139ff75 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/erase_iter_iter.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// iterator erase(const_iterator first, const_iterator last);
+// iterator erase(const_iterator first, const_iterator last); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a1[] = {1, 2, 3};
   {
     std::list<int> l1(a1, a1 + 3);
@@ -81,5 +81,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
index 8bd01c940d95..4475d27a7e73 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_initializer_list.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// iterator insert(const_iterator p, initializer_list<value_type> il);
+// iterator insert(const_iterator p, initializer_list<value_type> il); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> d(10, 1);
     std::list<int>::iterator i = d.insert(std::next(d.cbegin(), 2), {3, 4, 5, 6});
@@ -62,5 +62,14 @@ int main(int, char**) {
     assert(*i++ == 1);
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
index bab125ca6209..27db218511aa 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_iter_iter.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // template <InputIterator Iter>
-//   iterator insert(const_iterator position, Iter first, Iter last);
+//   iterator insert(const_iterator position, Iter first, Iter last); // constexpr since C++26
 
 #include <list>
 #include <cstdlib>
@@ -21,7 +21,7 @@
 #include "count_new.h"
 
 template <class List>
-void test() {
+TEST_CONSTEXPR_CXX26 void test() {
   int a1[] = {1, 2, 3};
   List l1;
   typename List::iterator i = l1.insert(l1.begin(), a1, a1 + 3);
@@ -53,36 +53,47 @@ void test() {
   assert(*i == 3);
 
 #if !defined(TEST_HAS_NO_EXCEPTIONS) && !defined(DISABLE_NEW_COUNT)
-  globalMemCounter.throw_after = 2;
-  int save_count               = globalMemCounter.outstanding_new;
-  try {
-    i = l1.insert(i, a2, a2 + 3);
-    assert(false);
-  } catch (...) {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    globalMemCounter.throw_after = 2;
+    int save_count               = globalMemCounter.outstanding_new;
+    try {
+      i = l1.insert(i, a2, a2 + 3);
+      assert(false);
+    } catch (...) {
+    }
+    assert(globalMemCounter.checkOutstandingNewEq(save_count));
+    assert(l1.size() == 6);
+    assert(std::distance(l1.begin(), l1.end()) == 6);
+    i = l1.begin();
+    assert(*i == 1);
+    ++i;
+    assert(*i == 2);
+    ++i;
+    assert(*i == 4);
+    ++i;
+    assert(*i == 5);
+    ++i;
+    assert(*i == 6);
+    ++i;
+    assert(*i == 3);
   }
-  assert(globalMemCounter.checkOutstandingNewEq(save_count));
-  assert(l1.size() == 6);
-  assert(std::distance(l1.begin(), l1.end()) == 6);
-  i = l1.begin();
-  assert(*i == 1);
-  ++i;
-  assert(*i == 2);
-  ++i;
-  assert(*i == 4);
-  ++i;
-  assert(*i == 5);
-  ++i;
-  assert(*i == 6);
-  ++i;
-  assert(*i == 3);
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   test<std::list<int> >();
 #if TEST_STD_VER >= 11
   test<std::list<int, min_allocator<int>>>();
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
index 8bb513208eb7..7d7b2f158a60 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_rvalue.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// iterator insert(const_iterator position, value_type&& x);
+// iterator insert(const_iterator position, value_type&& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly> l1;
     l1.insert(l1.cend(), MoveOnly(1));
@@ -41,5 +41,14 @@ int main(int, char**) {
     assert(l1.back() == MoveOnly(1));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
index 32ee7a73406d..1056d997f9d8 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_size_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// iterator insert(const_iterator position, size_type n, const value_type& x);
+// iterator insert(const_iterator position, size_type n, const value_type& x); // constexpr since C++26
 
 // UNSUPPORTED: sanitizer-new-delete
 
@@ -21,7 +21,7 @@
 #include "test_macros.h"
 
 template <class List>
-void test() {
+TEST_CONSTEXPR_CXX26 void test() {
   int a1[] = {1, 2, 3};
   int a2[] = {1, 4, 4, 4, 4, 4, 2, 3};
   List l1(a1, a1 + 3);
@@ -29,23 +29,34 @@ void test() {
   assert(i == std::next(l1.begin()));
   assert(l1 == List(a2, a2 + 8));
 #ifndef TEST_HAS_NO_EXCEPTIONS
-  globalMemCounter.throw_after = 4;
-  int save_count               = globalMemCounter.outstanding_new;
-  try {
-    i = l1.insert(i, 5, 5);
-    assert(false);
-  } catch (...) {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    globalMemCounter.throw_after = 4;
+    int save_count               = globalMemCounter.outstanding_new;
+    try {
+      i = l1.insert(i, 5, 5);
+      assert(false);
+    } catch (...) {
+    }
+    assert(globalMemCounter.checkOutstandingNewEq(save_count));
+    assert(l1 == List(a2, a2 + 8));
   }
-  assert(globalMemCounter.checkOutstandingNewEq(save_count));
-  assert(l1 == List(a2, a2 + 8));
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   test<std::list<int> >();
 #if TEST_STD_VER >= 11
   test<std::list<int, min_allocator<int>>>();
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
index 129fe05cb39d..615bb5bb2b42 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_iter_value.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// iterator insert(const_iterator position, const value_type& x);
+// iterator insert(const_iterator position, const value_type& x); // constexpr since C++26
 
 #include <list>
 #include <cstdlib>
@@ -19,7 +19,7 @@
 #include "count_new.h"
 
 template <class List>
-void test() {
+TEST_CONSTEXPR_CXX26 void test() {
   int a1[] = {1, 2, 3};
   int a2[] = {1, 4, 2, 3};
   List l1(a1, a1 + 3);
@@ -30,23 +30,34 @@ void test() {
   assert(l1 == List(a2, a2 + 4));
 
 #if !defined(TEST_HAS_NO_EXCEPTIONS) && !defined(DISABLE_NEW_COUNT)
-  globalMemCounter.throw_after = 0;
-  int save_count               = globalMemCounter.outstanding_new;
-  try {
-    i = l1.insert(i, 5);
-    assert(false);
-  } catch (...) {
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    globalMemCounter.throw_after = 0;
+    int save_count               = globalMemCounter.outstanding_new;
+    try {
+      i = l1.insert(i, 5);
+      assert(false);
+    } catch (...) {
+    }
+    assert(globalMemCounter.checkOutstandingNewEq(save_count));
+    assert(l1 == List(a2, a2 + 4));
   }
-  assert(globalMemCounter.checkOutstandingNewEq(save_count));
-  assert(l1 == List(a2, a2 + 4));
 #endif
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   test<std::list<int> >();
 #if TEST_STD_VER >= 11
   test<std::list<int, min_allocator<int>>>();
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_range.pass.cpp
index eb3937eb8f9e..5908d40d0cc9 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/insert_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/insert_range.pass.cpp
@@ -6,12 +6,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-steps): -fconstexpr-steps=20000000
+// ADDITIONAL_COMPILE_FLAGS(has-fconstexpr-ops-limit): -fconstexpr-ops-limit=80000000
+
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr iterator insert_range(const_iterator position, R&& rg); // C++23
+//   constexpr iterator insert_range(const_iterator position, R&& rg); // C++23; constexpr since C++26
 
 #include <list>
+#include <type_traits>
 
 #include "../../insert_range_sequence_containers.h"
 #include "test_macros.h"
@@ -21,7 +25,7 @@
 //   {empty/one-element/full} container at the {beginning/middle/end});
 // - inserting move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_insert_range<std::list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +35,19 @@ int main(int, char**) {
   });
   test_sequence_insert_range_move_only<std::list>();
 
-  test_insert_range_exception_safety_throwing_copy<std::list>();
-  test_insert_range_exception_safety_throwing_allocator<std::list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_insert_range_exception_safety_throwing_copy<std::list>();
+    test_insert_range_exception_safety_throwing_allocator<std::list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
index aaa225b14776..5bbac428d8d5 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_back.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void pop_back();
+// void pop_back(); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a[] = {1, 2, 3};
     std::list<int> c(a, a + 3);
@@ -40,5 +40,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
index 33b8ff35c524..74b6a1cc319b 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/pop_front.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void pop_front();
+// void pop_front(); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a[] = {1, 2, 3};
     std::list<int> c(a, a + 3);
@@ -40,5 +40,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/prepend_range.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/prepend_range.pass.cpp
index d5e4d4fabb76..41f7061c09d2 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/prepend_range.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/prepend_range.pass.cpp
@@ -9,9 +9,10 @@
 // UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
 
 // template<container-compatible-range<T> R>
-//   constexpr void prepend_range(R&& rg); // C++23
+//   constexpr void prepend_range(R&& rg); // C++23; constexpr since C++26
 
 #include <list>
+#include <type_traits>
 
 #include "../../insert_range_sequence_containers.h"
 #include "test_macros.h"
@@ -21,7 +22,7 @@
 //   {empty/one-element/full} container);
 // - prepending move-only elements;
 // - an exception is thrown when copying the elements or when allocating new elements.
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   static_assert(test_constraints_prepend_range<std::list, int, double>());
 
   for_all_iterators_and_allocators<int, const int*>([]<class Iter, class Sent, class Alloc>() {
@@ -31,8 +32,19 @@ int main(int, char**) {
   });
   test_sequence_prepend_range_move_only<std::list>();
 
-  test_prepend_range_exception_safety_throwing_copy<std::list>();
-  test_prepend_range_exception_safety_throwing_allocator<std::list, int>();
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    test_prepend_range_exception_safety_throwing_copy<std::list>();
+    test_prepend_range_exception_safety_throwing_allocator<std::list, int>();
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
index 582f4a200ac2..3ac9a60e7901 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void push_back(const value_type& x);
+// void push_back(const value_type& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> c;
     for (int i = 0; i < 5; ++i)
@@ -34,5 +34,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
index 6a31d81d694f..764dd7da1832 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_back_rvalue.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// void push_back(value_type&& x);
+// void push_back(value_type&& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly> l1;
     l1.push_back(MoveOnly(1));
@@ -41,5 +41,14 @@ int main(int, char**) {
     assert(l1.back() == MoveOnly(2));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
index 3b5f74a217a2..7ec18e841822 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void push_front(const value_type& x);
+// void push_front(const value_type& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<int> c;
     for (int i = 0; i < 5; ++i)
@@ -34,5 +34,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
index 0d41b8fd8553..930b6af5f243 100644
--- a/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.modifiers/push_front_rvalue.pass.cpp
@@ -10,7 +10,7 @@
 
 // <list>
 
-// void push_front(value_type&& x);
+// void push_front(value_type&& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,7 +19,7 @@
 #include "MoveOnly.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     std::list<MoveOnly> l1;
     l1.push_front(MoveOnly(1));
@@ -41,5 +41,14 @@ int main(int, char**) {
     assert(l1.back() == MoveOnly(1));
   }
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
index 7f82f65fd493..19ea940cb2a0 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/merge.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void merge(list& x);
+// void merge(list& x); // constexpr since C++26
 // If (addressof(x) == this) does nothing; otherwise ...
 
 #include <list>
@@ -17,7 +17,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {1, 3, 7, 9, 10};
     int a2[] = {0, 2, 4, 5, 6, 8, 11};
@@ -49,5 +49,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
index 13241909c6e5..974700926db6 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/merge_comp.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// template <class Compare> void merge(list& x, Compare comp);
+// template <class Compare> void merge(list& x, Compare comp); // constexpr since C++26
 // If (addressof(x) == this) does nothing; otherwise ...
 
 #include <list>
@@ -18,7 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {10, 9, 7, 3, 1};
     int a2[] = {11, 8, 6, 5, 4, 2, 0};
@@ -49,5 +49,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
index 238ea9b69ea2..9bf677b8745c 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/remove.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // void      remove(const value_type& value); // pre-c++20
-// size_type remove(const value_type& value); // c++20 and later
+// size_type remove(const value_type& value); // c++20 and later; constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,22 +18,22 @@
 #include "min_allocator.h"
 
 struct S {
-  S(int i) : i_(new int(i)) {}
-  S(const S& rhs) : i_(new int(*rhs.i_)) {}
-  S& operator=(const S& rhs) {
+  TEST_CONSTEXPR_CXX20 S(int i) : i_(new int(i)) {}
+  TEST_CONSTEXPR_CXX20 S(const S& rhs) : i_(new int(*rhs.i_)) {}
+  TEST_CONSTEXPR_CXX14 S& operator=(const S& rhs) {
     *i_ = *rhs.i_;
     return *this;
   }
-  ~S() {
+  TEST_CONSTEXPR_CXX20 ~S() {
     delete i_;
     i_ = NULL;
   }
-  bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
-  int get() const { return *i_; }
+  TEST_CONSTEXPR bool operator==(const S& rhs) const { return *i_ == *rhs.i_; }
+  TEST_CONSTEXPR int get() const { return *i_; }
   int* i_;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {1, 2, 4};
@@ -101,5 +101,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
index 510cb361142b..c7ee09530ed9 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/remove_if.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // template <class Pred> void      remove_if(Pred pred); // before C++20
-// template <class Pred> size_type remove_if(Pred pred); // c++20 and later
+// template <class Pred> size_type remove_if(Pred pred); // c++20 and later; constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -19,22 +19,22 @@
 #include "min_allocator.h"
 #include "counting_predicates.h"
 
-bool even(int i) { return i % 2 == 0; }
+TEST_CONSTEXPR bool even(int i) { return i % 2 == 0; }
 
-bool g(int i) { return i < 3; }
+TEST_CONSTEXPR bool g(int i) { return i < 3; }
 
 struct PredLWG526 {
-  PredLWG526(int i) : i_(i) {}
-  ~PredLWG526() { i_ = -32767; }
-  bool operator()(const PredLWG526& p) const { return p.i_ == i_; }
+  TEST_CONSTEXPR_CXX20 PredLWG526(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~PredLWG526() { i_ = -32767; }
+  TEST_CONSTEXPR bool operator()(const PredLWG526& p) const { return p.i_ == i_; }
 
-  bool operator==(int i) const { return i == i_; }
+  TEST_CONSTEXPR bool operator==(int i) const { return i == i_; }
   int i_;
 };
 
 typedef unary_counting_predicate<bool (*)(int), int> Predicate;
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {1, 2, 3, 4};
     int a2[] = {3, 4};
@@ -92,5 +92,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
index 5b91ad0224be..43e894f5e83a 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/reverse.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void reverse();
+// void reverse(); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
@@ -34,5 +34,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
index 892419f6ac96..34ead09110a3 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/sort.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void sort();
+// void sort(); // constexpr since C++26
 
 #include <algorithm>
 #include <list>
@@ -58,7 +58,7 @@ void test_stable(int N) {
   }
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11};
@@ -76,8 +76,19 @@ int main(int, char**) {
   }
 #endif
 
-  for (int i = 0; i < 40; ++i)
-    test_stable(i);
+  if (!TEST_IS_CONSTANT_EVALUATED) {
+    for (int i = 0; i < 40; ++i)
+      test_stable(i);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
index 499702281991..a24f187f4b4e 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/sort_comp.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// template <class Compare> sort(Compare comp);
+// template <class Compare> sort(Compare comp); // constexpr since C++26
 
 #include <list>
 #include <functional>
@@ -76,7 +76,7 @@ void test_stable(int N) {
   }
 }
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
     int a2[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
@@ -85,37 +85,48 @@ int main(int, char**) {
     assert(c1 == std::list<int>(a2, a2 + sizeof(a2) / sizeof(a2[0])));
   }
 
+  if (!TEST_IS_CONSTANT_EVALUATED) {
 //  Test with throwing comparison; make sure that nothing is lost.
 //  This is (sort of) LWG #2824
 #ifndef TEST_HAS_NO_EXCEPTIONS
-  {
-    int a1[]     = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
-    const int sz = sizeof(a1) / sizeof(a1[0]);
-    for (int i = 0; i < 10; ++i) {
-      std::list<int> c1(a1, a1 + sz);
-      try {
-        throwingLess<int> comp(i);
-        c1.sort(std::cref(comp));
-      } catch (int) {
+    {
+      int a1[]     = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
+      const int sz = sizeof(a1) / sizeof(a1[0]);
+      for (int i = 0; i < 10; ++i) {
+        std::list<int> c1(a1, a1 + sz);
+        try {
+          throwingLess<int> comp(i);
+          c1.sort(std::cref(comp));
+        } catch (int) {
+        }
+        assert((c1.size() == sz));
+        assert((std::is_permutation(c1.begin(), c1.end(), a1)));
       }
-      assert((c1.size() == sz));
-      assert((std::is_permutation(c1.begin(), c1.end(), a1)));
     }
-  }
 #endif
 
 #if TEST_STD_VER >= 11
-  {
-    int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
-    int a2[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
-    std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
-    c1.sort(std::greater<int>());
-    assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
-  }
+    {
+      int a1[] = {4, 8, 1, 0, 5, 7, 2, 3, 6, 11, 10, 9};
+      int a2[] = {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0};
+      std::list<int, min_allocator<int>> c1(a1, a1 + sizeof(a1) / sizeof(a1[0]));
+      c1.sort(std::greater<int>());
+      assert((c1 == std::list<int, min_allocator<int>>(a2, a2 + sizeof(a2) / sizeof(a2[0]))));
+    }
 #endif
 
-  for (int i = 0; i < 40; ++i)
-    test_stable(i);
+    for (int i = 0; i < 40; ++i)
+      test_stable(i);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
 
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
index 4b40876e3bb7..0f1cfefab34e 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void splice(const_iterator position, list& x);
+// void splice(const_iterator position, list& x); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a1[] = {1, 2, 3};
   int a2[] = {4, 5, 6};
   {
@@ -780,5 +780,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
index db71fe17a06e..38dce58dc390 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void splice(const_iterator position, list<T,Allocator>& x, iterator i);
+// void splice(const_iterator position, list<T,Allocator>& x, iterator i); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a1[] = {1, 2, 3};
   int a2[] = {4, 5, 6};
   {
@@ -334,5 +334,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
index b77b6a26440d..8fca21c81c66 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/splice_pos_list_iter_iter.pass.cpp
@@ -8,7 +8,7 @@
 
 // <list>
 
-// void splice(const_iterator position, list& x, iterator first, iterator last);
+// void splice(const_iterator position, list& x, iterator first, iterator last); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -16,7 +16,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   int a1[] = {1, 2, 3};
   int a2[] = {4, 5, 6};
   {
@@ -214,5 +214,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
index c08e348218f9..c2fa54f42553 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/unique.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // void      unique(); // before C++20
-// size_type unique(); // C++20 and later
+// size_type unique(); // C++20 and later; constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
@@ -46,5 +46,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp b/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
index 1d3a8e0c426a..830e54a3288d 100644
--- a/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.ops/unique_pred.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // template <class BinaryPred> void      unique(BinaryPred pred); // before C++20
-// template <class BinaryPred> size_type unique(BinaryPred pred); // C++20 and later
+// template <class BinaryPred> size_type unique(BinaryPred pred); // C++20 and later; constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -18,18 +18,18 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 
-bool g(int x, int y) { return x == y; }
+TEST_CONSTEXPR bool g(int x, int y) { return x == y; }
 
 struct PredLWG526 {
-  PredLWG526(int i) : i_(i) {}
-  ~PredLWG526() { i_ = -32767; }
-  bool operator()(const PredLWG526& lhs, const PredLWG526& rhs) const { return lhs.i_ == rhs.i_; }
+  TEST_CONSTEXPR_CXX20 PredLWG526(int i) : i_(i) {}
+  TEST_CONSTEXPR_CXX20 ~PredLWG526() { i_ = -32767; }
+  TEST_CONSTEXPR bool operator()(const PredLWG526& lhs, const PredLWG526& rhs) const { return lhs.i_ == rhs.i_; }
 
-  bool operator==(int i) const { return i == i_; }
+  TEST_CONSTEXPR bool operator==(int i) const { return i == i_; }
   int i_;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {2, 1, 1, 4, 4, 4, 4, 3, 3};
     int a2[] = {2, 1, 4, 3};
@@ -75,5 +75,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp b/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
index 1e9c71131d80..32efddb06920 100644
--- a/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.special/swap.pass.cpp
@@ -9,7 +9,7 @@
 // <list>
 
 // template <class T, class Alloc>
-//   void swap(list<T,Alloc>& x, list<T,Alloc>& y);
+//   void swap(list<T,Alloc>& x, list<T,Alloc>& y); // constexpr since C++26
 
 #include <list>
 #include <cassert>
@@ -17,7 +17,7 @@
 #include "test_allocator.h"
 #include "min_allocator.h"
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     int a1[] = {1, 3, 7, 9, 10};
     int a2[] = {0, 2, 4, 5, 6, 8, 11};
@@ -133,5 +133,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp b/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
index a4b1622a04be..037c7d07c4cb 100644
--- a/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
+++ b/libcxx/test/std/containers/sequences/list/list.special/swap_noexcept.pass.cpp
@@ -12,7 +12,7 @@
 
 // void swap(list& c)
 //     noexcept(!allocator_type::propagate_on_container_swap::value ||
-//              __is_nothrow_swappable<allocator_type>::value);
+//              __is_nothrow_swappable<allocator_type>::value); // constexpr since C++26
 //
 //  In C++17, the standard says that swap shall have:
 //     noexcept(allocator_traits<Allocator>::is_always_equal::value);
@@ -52,7 +52,7 @@ struct some_alloc2 {
   typedef std::true_type is_always_equal;
 };
 
-int main(int, char**) {
+TEST_CONSTEXPR_CXX26 bool test() {
   {
     typedef std::list<MoveOnly> C;
     static_assert(noexcept(swap(std::declval<C&>(), std::declval<C&>())), "");
@@ -84,5 +84,14 @@ int main(int, char**) {
   }
 #endif
 
+  return true;
+}
+
+int main(int, char**) {
+  assert(test());
+#if TEST_STD_VER >= 26
+  static_assert(test());
+#endif
+
   return 0;
 }
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/list.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/list.version.compile.pass.cpp
index 9fd638087fce..d10c61c0e9cf 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/list.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/list.version.compile.pass.cpp
@@ -24,6 +24,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -54,6 +58,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should not be defined before c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -87,6 +95,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++17"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -126,6 +138,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should not be defined before c++23"
 #  endif
@@ -171,6 +187,10 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++23"
 #  endif
@@ -219,6 +239,13 @@
 #    error "__cpp_lib_allocator_traits_is_always_equal should have the value 201411L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_list != 202502L
+#    error "__cpp_lib_constexpr_list should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_containers_ranges
 #    error "__cpp_lib_containers_ranges should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index a13edacd1e46..e4fe9f994e2e 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -208,6 +208,10 @@
 #    error "__cpp_lib_constexpr_iterator should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should not be defined before c++20"
 #  endif
@@ -1100,6 +1104,10 @@
 #    error "__cpp_lib_constexpr_iterator should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should not be defined before c++20"
 #  endif
@@ -2094,6 +2102,10 @@
 #    error "__cpp_lib_constexpr_iterator should not be defined before c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should not be defined before c++20"
 #  endif
@@ -3334,6 +3346,10 @@
 #    error "__cpp_lib_constexpr_iterator should have the value 201811L in c++20"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should be defined in c++20"
 #  endif
@@ -4790,6 +4806,10 @@
 #    error "__cpp_lib_constexpr_iterator should have the value 201811L in c++23"
 #  endif
 
+#  ifdef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should be defined in c++23"
 #  endif
@@ -6468,6 +6488,13 @@
 #    error "__cpp_lib_constexpr_iterator should have the value 201811L in c++26"
 #  endif
 
+#  ifndef __cpp_lib_constexpr_list
+#    error "__cpp_lib_constexpr_list should be defined in c++26"
+#  endif
+#  if __cpp_lib_constexpr_list != 202502L
+#    error "__cpp_lib_constexpr_list should have the value 202502L in c++26"
+#  endif
+
 #  ifndef __cpp_lib_constexpr_memory
 #    error "__cpp_lib_constexpr_memory should be defined in c++26"
 #  endif
diff --git a/libcxx/test/support/min_allocator.h b/libcxx/test/support/min_allocator.h
index 3b7d12af24ce..16775649f55c 100644
--- a/libcxx/test/support/min_allocator.h
+++ b/libcxx/test/support/min_allocator.h
@@ -22,384 +22,381 @@
 #include "test_macros.h"
 
 template <class T>
-class bare_allocator
-{
+class bare_allocator {
 public:
-    typedef T value_type;
+  typedef T value_type;
 
-    bare_allocator() TEST_NOEXCEPT {}
+  bare_allocator() TEST_NOEXCEPT {}
 
-    template <class U>
-    bare_allocator(bare_allocator<U>) TEST_NOEXCEPT {}
+  template <class U>
+  bare_allocator(bare_allocator<U>) TEST_NOEXCEPT {}
 
-    T* allocate(std::size_t n)
-    {
-        return static_cast<T*>(::operator new(n*sizeof(T)));
-    }
+  T* allocate(std::size_t n) { return static_cast<T*>(::operator new(n * sizeof(T))); }
 
-    void deallocate(T* p, std::size_t)
-    {
-        return ::operator delete(static_cast<void*>(p));
-    }
+  void deallocate(T* p, std::size_t) { return ::operator delete(static_cast<void*>(p)); }
 
-    friend bool operator==(bare_allocator, bare_allocator) {return true;}
-    friend bool operator!=(bare_allocator x, bare_allocator y) {return !(x == y);}
+  friend bool operator==(bare_allocator, bare_allocator) { return true; }
+  friend bool operator!=(bare_allocator x, bare_allocator y) { return !(x == y); }
 };
 
-
 template <class T>
-class no_default_allocator
-{
+class no_default_allocator {
 #if TEST_STD_VER >= 11
-    no_default_allocator() = delete;
+  no_default_allocator() = delete;
 #else
-    no_default_allocator();
+  no_default_allocator();
 #endif
-    struct construct_tag {};
-    explicit no_default_allocator(construct_tag) {}
+  struct construct_tag {};
+  TEST_CONSTEXPR_CXX20 explicit no_default_allocator(construct_tag) {}
 
 public:
-    static no_default_allocator create() {
-      construct_tag tag;
-      return no_default_allocator(tag);
-    }
+  TEST_CONSTEXPR_CXX20 static no_default_allocator create() {
+    construct_tag tag;
+    return no_default_allocator(tag);
+  }
 
 public:
-    typedef T value_type;
+  typedef T value_type;
 
-    template <class U>
-    no_default_allocator(no_default_allocator<U>) TEST_NOEXCEPT {}
+  template <class U>
+  TEST_CONSTEXPR_CXX20 no_default_allocator(no_default_allocator<U>) TEST_NOEXCEPT {}
 
-    T* allocate(std::size_t n)
-    {
-        return static_cast<T*>(::operator new(n*sizeof(T)));
-    }
+  TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n) { return static_cast<T*>(std::allocator<T>().allocate(n)); }
 
-    void deallocate(T* p, std::size_t)
-    {
-        return ::operator delete(static_cast<void*>(p));
-    }
+  TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
 
-    friend bool operator==(no_default_allocator, no_default_allocator) {return true;}
-    friend bool operator!=(no_default_allocator x, no_default_allocator y) {return !(x == y);}
+  friend TEST_CONSTEXPR bool operator==(no_default_allocator, no_default_allocator) { return true; }
+  friend TEST_CONSTEXPR bool operator!=(no_default_allocator x, no_default_allocator y) { return !(x == y); }
 };
 
 struct malloc_allocator_base {
-    static std::size_t outstanding_bytes;
-    static std::size_t alloc_count;
-    static std::size_t dealloc_count;
-    static bool disable_default_constructor;
-
-    static std::size_t outstanding_alloc() {
-      assert(alloc_count >= dealloc_count);
-      return (alloc_count - dealloc_count);
-    }
-
-    static void reset() {
-        assert(outstanding_alloc() == 0);
-        disable_default_constructor = false;
-        outstanding_bytes = 0;
-        alloc_count = 0;
-        dealloc_count = 0;
-    }
+  static std::size_t outstanding_bytes;
+  static std::size_t alloc_count;
+  static std::size_t dealloc_count;
+  static bool disable_default_constructor;
+
+  static std::size_t outstanding_alloc() {
+    assert(alloc_count >= dealloc_count);
+    return (alloc_count - dealloc_count);
+  }
+
+  static void reset() {
+    assert(outstanding_alloc() == 0);
+    disable_default_constructor = false;
+    outstanding_bytes           = 0;
+    alloc_count                 = 0;
+    dealloc_count               = 0;
+  }
 };
 
-size_t malloc_allocator_base::outstanding_bytes = 0;
-size_t malloc_allocator_base::alloc_count = 0;
-size_t malloc_allocator_base::dealloc_count = 0;
+size_t malloc_allocator_base::outstanding_bytes         = 0;
+size_t malloc_allocator_base::alloc_count               = 0;
+size_t malloc_allocator_base::dealloc_count             = 0;
 bool malloc_allocator_base::disable_default_constructor = false;
 
-
 template <class T>
-class malloc_allocator : public malloc_allocator_base
-{
+class malloc_allocator : public malloc_allocator_base {
 public:
-    typedef T value_type;
-
-    malloc_allocator() TEST_NOEXCEPT { assert(!disable_default_constructor); }
-
-    template <class U>
-    malloc_allocator(malloc_allocator<U>) TEST_NOEXCEPT {}
-
-    T* allocate(std::size_t n)
-    {
-        const std::size_t nbytes = n*sizeof(T);
-        ++alloc_count;
-        outstanding_bytes += nbytes;
-        return static_cast<T*>(std::malloc(nbytes));
-    }
-
-    void deallocate(T* p, std::size_t n)
-    {
-        const std::size_t nbytes = n*sizeof(T);
-        ++dealloc_count;
-        outstanding_bytes -= nbytes;
-        std::free(static_cast<void*>(p));
-    }
-
-    friend bool operator==(malloc_allocator, malloc_allocator) {return true;}
-    friend bool operator!=(malloc_allocator x, malloc_allocator y) {return !(x == y);}
+  typedef T value_type;
+
+  malloc_allocator() TEST_NOEXCEPT { assert(!disable_default_constructor); }
+
+  template <class U>
+  malloc_allocator(malloc_allocator<U>) TEST_NOEXCEPT {}
+
+  T* allocate(std::size_t n) {
+    const std::size_t nbytes = n * sizeof(T);
+    ++alloc_count;
+    outstanding_bytes += nbytes;
+    return static_cast<T*>(std::malloc(nbytes));
+  }
+
+  void deallocate(T* p, std::size_t n) {
+    const std::size_t nbytes = n * sizeof(T);
+    ++dealloc_count;
+    outstanding_bytes -= nbytes;
+    std::free(static_cast<void*>(p));
+  }
+
+  friend bool operator==(malloc_allocator, malloc_allocator) { return true; }
+  friend bool operator!=(malloc_allocator x, malloc_allocator y) { return !(x == y); }
 };
 
 template <class T>
-struct cpp03_allocator : bare_allocator<T>
-{
-    typedef T value_type;
-    typedef value_type* pointer;
-
-    static bool construct_called;
-
-    // Returned value is not used but it's not prohibited.
-    pointer construct(pointer p, const value_type& val)
-    {
-        ::new(p) value_type(val);
-        construct_called = true;
-        return p;
-    }
-
-    std::size_t max_size() const
-    {
-        return UINT_MAX / sizeof(T);
-    }
+struct cpp03_allocator : bare_allocator<T> {
+  typedef T value_type;
+  typedef value_type* pointer;
+
+  static bool construct_called;
+
+  // Returned value is not used but it's not prohibited.
+  pointer construct(pointer p, const value_type& val) {
+    ::new (p) value_type(val);
+    construct_called = true;
+    return p;
+  }
+
+  std::size_t max_size() const { return UINT_MAX / sizeof(T); }
 };
-template <class T> bool cpp03_allocator<T>::construct_called = false;
+template <class T>
+bool cpp03_allocator<T>::construct_called = false;
 
 template <class T>
-struct cpp03_overload_allocator : bare_allocator<T>
-{
-    typedef T value_type;
-    typedef value_type* pointer;
-
-    static bool construct_called;
-
-    void construct(pointer p, const value_type& val)
-    {
-        construct(p, val, std::is_class<T>());
-    }
-    void construct(pointer p, const value_type& val, std::true_type)
-    {
-        ::new(p) value_type(val);
-        construct_called = true;
-    }
-    void construct(pointer p, const value_type& val, std::false_type)
-    {
-        ::new(p) value_type(val);
-        construct_called = true;
-    }
-
-    std::size_t max_size() const
-    {
-        return UINT_MAX / sizeof(T);
-    }
+struct cpp03_overload_allocator : bare_allocator<T> {
+  typedef T value_type;
+  typedef value_type* pointer;
+
+  static bool construct_called;
+
+  void construct(pointer p, const value_type& val) { construct(p, val, std::is_class<T>()); }
+  void construct(pointer p, const value_type& val, std::true_type) {
+    ::new (p) value_type(val);
+    construct_called = true;
+  }
+  void construct(pointer p, const value_type& val, std::false_type) {
+    ::new (p) value_type(val);
+    construct_called = true;
+  }
+
+  std::size_t max_size() const { return UINT_MAX / sizeof(T); }
 };
-template <class T> bool cpp03_overload_allocator<T>::construct_called = false;
+template <class T>
+bool cpp03_overload_allocator<T>::construct_called = false;
 
-template <class T, class = std::integral_constant<std::size_t, 0> > class min_pointer;
-template <class T, class ID> class min_pointer<const T, ID>;
-template <class ID> class min_pointer<void, ID>;
-template <class ID> class min_pointer<const void, ID>;
-template <class T> class min_allocator;
+template <class T, class = std::integral_constant<std::size_t, 0> >
+class min_pointer;
+template <class T, class ID>
+class min_pointer<const T, ID>;
+template <class ID>
+class min_pointer<void, ID>;
+template <class ID>
+class min_pointer<const void, ID>;
+template <class T>
+class min_allocator;
 
 template <class ID>
-class min_pointer<const void, ID>
-{
-    const void* ptr_;
+class min_pointer<const void, ID> {
+  const void* ptr_;
+
 public:
-    min_pointer() TEST_NOEXCEPT = default;
-    min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
-    template <class T>
-    min_pointer(min_pointer<T, ID> p) TEST_NOEXCEPT : ptr_(p.ptr_) {}
+  min_pointer() TEST_NOEXCEPT = default;
+  min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
+  template <class T>
+  min_pointer(min_pointer<T, ID> p) TEST_NOEXCEPT : ptr_(p.ptr_) {}
 
-    explicit operator bool() const {return ptr_ != nullptr;}
+  explicit operator bool() const { return ptr_ != nullptr; }
 
-    friend bool operator==(min_pointer x, min_pointer y) {return x.ptr_ == y.ptr_;}
-    friend bool operator!=(min_pointer x, min_pointer y) {return !(x == y);}
-    template <class U, class XID> friend class min_pointer;
+  friend bool operator==(min_pointer x, min_pointer y) { return x.ptr_ == y.ptr_; }
+  friend bool operator!=(min_pointer x, min_pointer y) { return !(x == y); }
+  template <class U, class XID>
+  friend class min_pointer;
 };
 
 template <class ID>
-class min_pointer<void, ID>
-{
-    void* ptr_;
+class min_pointer<void, ID> {
+  void* ptr_;
+
 public:
-    min_pointer() TEST_NOEXCEPT = default;
-    TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
-    template <class T,
-              class = typename std::enable_if
-                       <
-                            !std::is_const<T>::value
-                       >::type
-             >
-    TEST_CONSTEXPR_CXX14 min_pointer(min_pointer<T, ID> p) TEST_NOEXCEPT : ptr_(p.ptr_) {}
-
-    TEST_CONSTEXPR_CXX14 explicit operator bool() const {return ptr_ != nullptr;}
-
-    TEST_CONSTEXPR_CXX14 friend bool operator==(min_pointer x, min_pointer y) {return x.ptr_ == y.ptr_;}
-    TEST_CONSTEXPR_CXX14 friend bool operator!=(min_pointer x, min_pointer y) {return !(x == y);}
-    template <class U, class XID> friend class min_pointer;
+  min_pointer() TEST_NOEXCEPT = default;
+  TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
+  template <class T, class = typename std::enable_if< !std::is_const<T>::value >::type >
+  TEST_CONSTEXPR_CXX14 min_pointer(min_pointer<T, ID> p) TEST_NOEXCEPT : ptr_(p.ptr_) {}
+
+  TEST_CONSTEXPR_CXX14 explicit operator bool() const { return ptr_ != nullptr; }
+
+  TEST_CONSTEXPR_CXX14 friend bool operator==(min_pointer x, min_pointer y) { return x.ptr_ == y.ptr_; }
+  TEST_CONSTEXPR_CXX14 friend bool operator!=(min_pointer x, min_pointer y) { return !(x == y); }
+  template <class U, class XID>
+  friend class min_pointer;
 };
 
 template <class T, class ID>
-class min_pointer
-{
-    T* ptr_;
+class min_pointer {
+  T* ptr_;
+
+  TEST_CONSTEXPR_CXX14 explicit min_pointer(T* p) TEST_NOEXCEPT : ptr_(p) {}
 
-    TEST_CONSTEXPR_CXX14 explicit min_pointer(T* p) TEST_NOEXCEPT : ptr_(p) {}
 public:
-    min_pointer() TEST_NOEXCEPT = default;
-    TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
-    TEST_CONSTEXPR_CXX14 explicit min_pointer(min_pointer<void, ID> p) TEST_NOEXCEPT : ptr_(static_cast<T*>(p.ptr_)) {}
-
-    TEST_CONSTEXPR_CXX14 explicit operator bool() const {return ptr_ != nullptr;}
-
-    typedef std::ptrdiff_t difference_type;
-    typedef T& reference;
-    typedef T* pointer;
-    typedef T value_type;
-    typedef std::random_access_iterator_tag iterator_category;
-
-    TEST_CONSTEXPR_CXX14 reference operator*() const {return *ptr_;}
-    TEST_CONSTEXPR_CXX14 pointer operator->() const {return ptr_;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator++() {++ptr_; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer operator++(int) {min_pointer tmp(*this); ++ptr_; return tmp;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator--() {--ptr_; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer operator--(int) {min_pointer tmp(*this); --ptr_; return tmp;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator+=(difference_type n) {ptr_ += n; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer& operator-=(difference_type n) {ptr_ -= n; return *this;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n) const
-    {
-        min_pointer tmp(*this);
-        tmp += n;
-        return tmp;
-    }
-
-    friend TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n, min_pointer x)
-    {
-        return x + n;
-    }
-
-    TEST_CONSTEXPR_CXX14 min_pointer operator-(difference_type n) const
-    {
-        min_pointer tmp(*this);
-        tmp -= n;
-        return tmp;
-    }
-
-    friend TEST_CONSTEXPR_CXX14 difference_type operator-(min_pointer x, min_pointer y)
-    {
-        return x.ptr_ - y.ptr_;
-    }
-
-    TEST_CONSTEXPR_CXX14 reference operator[](difference_type n) const {return ptr_[n];}
-
-    friend TEST_CONSTEXPR_CXX14 bool operator< (min_pointer x, min_pointer y) {return x.ptr_ < y.ptr_;}
-    friend TEST_CONSTEXPR_CXX14 bool operator> (min_pointer x, min_pointer y) {return y < x;}
-    friend TEST_CONSTEXPR_CXX14 bool operator<=(min_pointer x, min_pointer y) {return !(y < x);}
-    friend TEST_CONSTEXPR_CXX14 bool operator>=(min_pointer x, min_pointer y) {return !(x < y);}
-
-    static TEST_CONSTEXPR_CXX14 min_pointer pointer_to(T& t) {return min_pointer(std::addressof(t));}
-
-    friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, min_pointer y) {return x.ptr_ == y.ptr_;}
-    friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, min_pointer y) {return !(x == y);}
-    template <class U, class XID> friend class min_pointer;
-    template <class U> friend class min_allocator;
+  min_pointer() TEST_NOEXCEPT = default;
+  TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) TEST_NOEXCEPT : ptr_(nullptr) {}
+  TEST_CONSTEXPR_CXX14 explicit min_pointer(min_pointer<void, ID> p) TEST_NOEXCEPT : ptr_(static_cast<T*>(p.ptr_)) {}
+
+  TEST_CONSTEXPR_CXX14 explicit operator bool() const { return ptr_ != nullptr; }
+
+  typedef std::ptrdiff_t difference_type;
+  typedef T& reference;
+  typedef T* pointer;
+  typedef T value_type;
+  typedef std::random_access_iterator_tag iterator_category;
+
+  TEST_CONSTEXPR_CXX14 reference operator*() const { return *ptr_; }
+  TEST_CONSTEXPR_CXX14 pointer operator->() const { return ptr_; }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator++() {
+    ++ptr_;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer operator++(int) {
+    min_pointer tmp(*this);
+    ++ptr_;
+    return tmp;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator--() {
+    --ptr_;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer operator--(int) {
+    min_pointer tmp(*this);
+    --ptr_;
+    return tmp;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator+=(difference_type n) {
+    ptr_ += n;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer& operator-=(difference_type n) {
+    ptr_ -= n;
+    return *this;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n) const {
+    min_pointer tmp(*this);
+    tmp += n;
+    return tmp;
+  }
+
+  friend TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n, min_pointer x) { return x + n; }
+
+  TEST_CONSTEXPR_CXX14 min_pointer operator-(difference_type n) const {
+    min_pointer tmp(*this);
+    tmp -= n;
+    return tmp;
+  }
+
+  friend TEST_CONSTEXPR_CXX14 difference_type operator-(min_pointer x, min_pointer y) { return x.ptr_ - y.ptr_; }
+
+  TEST_CONSTEXPR_CXX14 reference operator[](difference_type n) const { return ptr_[n]; }
+
+  friend TEST_CONSTEXPR_CXX14 bool operator<(min_pointer x, min_pointer y) { return x.ptr_ < y.ptr_; }
+  friend TEST_CONSTEXPR_CXX14 bool operator>(min_pointer x, min_pointer y) { return y < x; }
+  friend TEST_CONSTEXPR_CXX14 bool operator<=(min_pointer x, min_pointer y) { return !(y < x); }
+  friend TEST_CONSTEXPR_CXX14 bool operator>=(min_pointer x, min_pointer y) { return !(x < y); }
+
+  static TEST_CONSTEXPR_CXX14 min_pointer pointer_to(T& t) { return min_pointer(std::addressof(t)); }
+
+  friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, min_pointer y) { return x.ptr_ == y.ptr_; }
+  friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, min_pointer y) { return !(x == y); }
+  template <class U, class XID>
+  friend class min_pointer;
+  template <class U>
+  friend class min_allocator;
 };
 
 template <class T, class ID>
-class min_pointer<const T, ID>
-{
-    const T* ptr_;
+class min_pointer<const T, ID> {
+  const T* ptr_;
+
+  TEST_CONSTEXPR_CXX14 explicit min_pointer(const T* p) : ptr_(p) {}
 
-    TEST_CONSTEXPR_CXX14 explicit min_pointer(const T* p) : ptr_(p) {}
 public:
-    min_pointer() TEST_NOEXCEPT = default;
-    TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) : ptr_(nullptr) {}
-    TEST_CONSTEXPR_CXX14 min_pointer(min_pointer<T, ID> p) : ptr_(p.ptr_) {}
-    TEST_CONSTEXPR_CXX14 explicit min_pointer(min_pointer<const void, ID> p) : ptr_(static_cast<const T*>(p.ptr_)) {}
-
-    TEST_CONSTEXPR_CXX14 explicit operator bool() const {return ptr_ != nullptr;}
-
-    typedef std::ptrdiff_t difference_type;
-    typedef const T& reference;
-    typedef const T* pointer;
-    typedef const T value_type;
-    typedef std::random_access_iterator_tag iterator_category;
-
-    TEST_CONSTEXPR_CXX14 reference operator*() const {return *ptr_;}
-    TEST_CONSTEXPR_CXX14 pointer operator->() const {return ptr_;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator++() {++ptr_; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer operator++(int) {min_pointer tmp(*this); ++ptr_; return tmp;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator--() {--ptr_; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer operator--(int) {min_pointer tmp(*this); --ptr_; return tmp;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer& operator+=(difference_type n) {ptr_ += n; return *this;}
-    TEST_CONSTEXPR_CXX14 min_pointer& operator-=(difference_type n) {ptr_ -= n; return *this;}
-
-    TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n) const
-    {
-        min_pointer tmp(*this);
-        tmp += n;
-        return tmp;
-    }
-
-    friend TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n, min_pointer x)
-    {
-        return x + n;
-    }
-
-    TEST_CONSTEXPR_CXX14 min_pointer operator-(difference_type n) const
-    {
-        min_pointer tmp(*this);
-        tmp -= n;
-        return tmp;
-    }
-
-    friend TEST_CONSTEXPR_CXX14 difference_type operator-(min_pointer x, min_pointer y)
-    {
-        return x.ptr_ - y.ptr_;
-    }
-
-    TEST_CONSTEXPR_CXX14 reference operator[](difference_type n) const {return ptr_[n];}
-
-    friend TEST_CONSTEXPR_CXX14 bool operator< (min_pointer x, min_pointer y) {return x.ptr_ < y.ptr_;}
-    friend TEST_CONSTEXPR_CXX14 bool operator> (min_pointer x, min_pointer y) {return y < x;}
-    friend TEST_CONSTEXPR_CXX14 bool operator<=(min_pointer x, min_pointer y) {return !(y < x);}
-    friend TEST_CONSTEXPR_CXX14 bool operator>=(min_pointer x, min_pointer y) {return !(x < y);}
-
-    static TEST_CONSTEXPR_CXX14 min_pointer pointer_to(const T& t) {return min_pointer(std::addressof(t));}
-
-    friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, min_pointer y) {return x.ptr_ == y.ptr_;}
-    friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, min_pointer y) {return x.ptr_ != y.ptr_;}
-    friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, std::nullptr_t) {return x.ptr_ == nullptr;}
-    friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, std::nullptr_t) {return x.ptr_ != nullptr;}
-    friend TEST_CONSTEXPR_CXX14 bool operator==(std::nullptr_t, min_pointer x) {return x.ptr_ == nullptr;}
-    friend TEST_CONSTEXPR_CXX14 bool operator!=(std::nullptr_t, min_pointer x) {return x.ptr_ != nullptr;}
-    template <class U, class XID> friend class min_pointer;
+  min_pointer() TEST_NOEXCEPT = default;
+  TEST_CONSTEXPR_CXX14 min_pointer(std::nullptr_t) : ptr_(nullptr) {}
+  TEST_CONSTEXPR_CXX14 min_pointer(min_pointer<T, ID> p) : ptr_(p.ptr_) {}
+  TEST_CONSTEXPR_CXX14 explicit min_pointer(min_pointer<const void, ID> p) : ptr_(static_cast<const T*>(p.ptr_)) {}
+
+  TEST_CONSTEXPR_CXX14 explicit operator bool() const { return ptr_ != nullptr; }
+
+  typedef std::ptrdiff_t difference_type;
+  typedef const T& reference;
+  typedef const T* pointer;
+  typedef const T value_type;
+  typedef std::random_access_iterator_tag iterator_category;
+
+  TEST_CONSTEXPR_CXX14 reference operator*() const { return *ptr_; }
+  TEST_CONSTEXPR_CXX14 pointer operator->() const { return ptr_; }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator++() {
+    ++ptr_;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer operator++(int) {
+    min_pointer tmp(*this);
+    ++ptr_;
+    return tmp;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator--() {
+    --ptr_;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer operator--(int) {
+    min_pointer tmp(*this);
+    --ptr_;
+    return tmp;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer& operator+=(difference_type n) {
+    ptr_ += n;
+    return *this;
+  }
+  TEST_CONSTEXPR_CXX14 min_pointer& operator-=(difference_type n) {
+    ptr_ -= n;
+    return *this;
+  }
+
+  TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n) const {
+    min_pointer tmp(*this);
+    tmp += n;
+    return tmp;
+  }
+
+  friend TEST_CONSTEXPR_CXX14 min_pointer operator+(difference_type n, min_pointer x) { return x + n; }
+
+  TEST_CONSTEXPR_CXX14 min_pointer operator-(difference_type n) const {
+    min_pointer tmp(*this);
+    tmp -= n;
+    return tmp;
+  }
+
+  friend TEST_CONSTEXPR_CXX14 difference_type operator-(min_pointer x, min_pointer y) { return x.ptr_ - y.ptr_; }
+
+  TEST_CONSTEXPR_CXX14 reference operator[](difference_type n) const { return ptr_[n]; }
+
+  friend TEST_CONSTEXPR_CXX14 bool operator<(min_pointer x, min_pointer y) { return x.ptr_ < y.ptr_; }
+  friend TEST_CONSTEXPR_CXX14 bool operator>(min_pointer x, min_pointer y) { return y < x; }
+  friend TEST_CONSTEXPR_CXX14 bool operator<=(min_pointer x, min_pointer y) { return !(y < x); }
+  friend TEST_CONSTEXPR_CXX14 bool operator>=(min_pointer x, min_pointer y) { return !(x < y); }
+
+  static TEST_CONSTEXPR_CXX14 min_pointer pointer_to(const T& t) { return min_pointer(std::addressof(t)); }
+
+  friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, min_pointer y) { return x.ptr_ == y.ptr_; }
+  friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, min_pointer y) { return x.ptr_ != y.ptr_; }
+  friend TEST_CONSTEXPR_CXX14 bool operator==(min_pointer x, std::nullptr_t) { return x.ptr_ == nullptr; }
+  friend TEST_CONSTEXPR_CXX14 bool operator!=(min_pointer x, std::nullptr_t) { return x.ptr_ != nullptr; }
+  friend TEST_CONSTEXPR_CXX14 bool operator==(std::nullptr_t, min_pointer x) { return x.ptr_ == nullptr; }
+  friend TEST_CONSTEXPR_CXX14 bool operator!=(std::nullptr_t, min_pointer x) { return x.ptr_ != nullptr; }
+  template <class U, class XID>
+  friend class min_pointer;
 };
 
 template <class T>
-class min_allocator
-{
+class min_allocator {
 public:
-    typedef T value_type;
-    typedef min_pointer<T> pointer;
+  typedef T value_type;
+  typedef min_pointer<T> pointer;
 
-    min_allocator() = default;
-    template <class U>
-    TEST_CONSTEXPR_CXX20 min_allocator(min_allocator<U>) {}
+  min_allocator() = default;
+  template <class U>
+  TEST_CONSTEXPR_CXX20 min_allocator(min_allocator<U>) {}
 
-    TEST_CONSTEXPR_CXX20 pointer allocate(std::size_t n) { return pointer(std::allocator<T>().allocate(n)); }
+  TEST_CONSTEXPR_CXX20 pointer allocate(std::size_t n) { return pointer(std::allocator<T>().allocate(n)); }
 
-    TEST_CONSTEXPR_CXX20 void deallocate(pointer p, std::size_t n) { std::allocator<T>().deallocate(p.ptr_, n); }
+  TEST_CONSTEXPR_CXX20 void deallocate(pointer p, std::size_t n) { std::allocator<T>().deallocate(p.ptr_, n); }
 
-    TEST_CONSTEXPR_CXX20 friend bool operator==(min_allocator, min_allocator) {return true;}
-    TEST_CONSTEXPR_CXX20 friend bool operator!=(min_allocator x, min_allocator y) {return !(x == y);}
+  TEST_CONSTEXPR_CXX20 friend bool operator==(min_allocator, min_allocator) { return true; }
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(min_allocator x, min_allocator y) { return !(x == y); }
 };
 
 template <class T>
@@ -427,25 +424,19 @@ template <class T>
 class explicit_allocator
 {
 public:
-    typedef T value_type;
+  typedef T value_type;
 
-    TEST_CONSTEXPR_CXX20 explicit_allocator() TEST_NOEXCEPT {}
+  TEST_CONSTEXPR_CXX20 explicit_allocator() TEST_NOEXCEPT {}
 
-    template <class U>
-    TEST_CONSTEXPR_CXX20 explicit explicit_allocator(explicit_allocator<U>) TEST_NOEXCEPT {}
+  template <class U>
+  TEST_CONSTEXPR_CXX20 explicit explicit_allocator(explicit_allocator<U>) TEST_NOEXCEPT {}
 
-    TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n)
-    {
-        return static_cast<T*>(std::allocator<T>().allocate(n));
-    }
+  TEST_CONSTEXPR_CXX20 T* allocate(std::size_t n) { return static_cast<T*>(std::allocator<T>().allocate(n)); }
 
-    TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n)
-    {
-        std::allocator<T>().deallocate(p, n);
-    }
+  TEST_CONSTEXPR_CXX20 void deallocate(T* p, std::size_t n) { std::allocator<T>().deallocate(p, n); }
 
-    TEST_CONSTEXPR_CXX20 friend bool operator==(explicit_allocator, explicit_allocator) {return true;}
-    TEST_CONSTEXPR_CXX20 friend bool operator!=(explicit_allocator x, explicit_allocator y) {return !(x == y);}
+  TEST_CONSTEXPR_CXX20 friend bool operator==(explicit_allocator, explicit_allocator) { return true; }
+  TEST_CONSTEXPR_CXX20 friend bool operator!=(explicit_allocator x, explicit_allocator y) { return !(x == y); }
 };
 
 template <class T>
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index b59c7fdaf0a3..de06b9dd1bee 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -372,6 +372,11 @@ feature_test_macros = [
             "values": {"c++20": 201811},
             "headers": ["iterator"],
         },
+        {
+            "name": "__cpp_lib_constexpr_list",
+            "values": {"c++26": 202502},
+            "headers": ["list"],
+        },
         {
             "name": "__cpp_lib_constexpr_memory",
             "values": {"c++20": 201811, "c++23": 202202},
-- 
cgit v1.2.3


From 6d785ca4218b18e77e39320bea7f8973c3ea2764 Mon Sep 17 00:00:00 2001
From: Ying Yi <ying.yi@sony.com>
Date: Wed, 18 Jun 2025 17:14:33 +0100
Subject: [Clang] Fix the clang/test/PCH/ignored-pch.c test. (#144737)

Change the test to check the exit status of the 'ls' command line
(instead of error message) since the error message is different when
running 'ls' command on the different Host machine.
---
 clang/test/PCH/ignored-pch.c | 58 +++++++++++++++++++++-----------------------
 1 file changed, 27 insertions(+), 31 deletions(-)

diff --git a/clang/test/PCH/ignored-pch.c b/clang/test/PCH/ignored-pch.c
index 5b64582cba61..c6ef3fe74cee 100644
--- a/clang/test/PCH/ignored-pch.c
+++ b/clang/test/PCH/ignored-pch.c
@@ -1,96 +1,96 @@
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -o %t.ll
-// RUN: ls %t.pch | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: ls %t.pch
+// RUN: ls %t.ll
 
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -o %t.pch
 // RUN: %clang %s -emit-ast -include-pch %t.pch -o %t.ll
-// RUN: ls %t.pch | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: ls %t.pch
+// RUN: ls %t.ll
 
 // Check that -ignore-pch causes -emit-pch and -include-pch options to be ignored.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH-ERROR %s
-// RUN: ls %t.ll 2>&1 | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -emit-ast %s -include-pch %t.pch -ignore-pch -o %t.ll
-// RUN: not ls %t.ll 2>&1 | FileCheck --check-prefix=CHECK-OBJ-ERROR %s
+// RUN: not ls %t.ll
 
 // Check that -ignore-pch works for multiple PCH related options.
 // Test with -building-pch-with-obj.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -building-pch-with-obj -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -Xclang -building-pch-with-obj -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH-ERROR %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fallow-pch-with-compiler-errors.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -fallow-pch-with-compiler-errors -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -Xclang -fallow-pch-with-compiler-errors -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH-ERROR %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fallow-pch-with-different-modules-cache-path.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -fallow-pch-with-different-modules-cache-path -o %t.pch
 // RUN: %clang -S -emit-llvm %s -ignore-pch -include-pch %t.pch -Xclang -fallow-pch-with-different-modules-cache-path -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH-ERROR %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fpch-codegen.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -fpch-codegen -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -fpch-codegen -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH-ERROR %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fpch-debuginfo.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -fpch-debuginfo -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -fpch-debuginfo -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fpch-instantiate-templates.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -fpch-instantiate-templates -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -fpch-instantiate-templates -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fno-pch-timestamp.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -fno-pch-timestamp -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -Xclang -fno-pch-timestamp -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -fno-validate-pch.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -fno-validate-pch -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -Xclang -fno-validate-pch -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -relocatable-pch.
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -relocatable-pch -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -relocatable-pch -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 // Test with -pch-through-hdrstop-create/-pch-through-hdrstop-use
 // RUN: rm -rf %t.pch %t.ll
 // RUN: %clang -x c-header %S/Inputs/ignored-pch.h -ignore-pch -Xclang -pch-through-hdrstop-create -o %t.pch
 // RUN: %clang -S -emit-llvm %s -include-pch %t.pch -ignore-pch -Xclang -pch-through-hdrstop-use -o %t.ll
-// RUN: not ls %t.pch 2>&1 | FileCheck --check-prefix=CHECK-PCH %s
-// RUN: ls %t.ll | FileCheck --check-prefix=CHECK-OBJ %s
+// RUN: not ls %t.pch
+// RUN: ls %t.ll
 
 
 // Test with AST dump output:
@@ -99,10 +99,6 @@
 // RUN: %clang %s -include-pch %t.pch -Xclang -ast-dump-all -c | FileCheck --check-prefix=CHECK-AST-PCH %s
 // RUN: %clang %s -include-pch %t.pch -ignore-pch -Xclang -ast-dump-all -c | FileCheck --check-prefix=CHECK-AST %s
 
-// CHECK-PCH: ignored-pch.c.{{.*}}.pch
-// CHECK-OBJ: ignored-pch.c.{{.*}}.ll
-// CHECK-PCH-ERROR: ignored-pch.c.{{.*}}.pch{{'?}}: No such file or directory
-// CHECK-OBJ-ERROR: ignored-pch.c.{{.*}}.ll{{'?}}: No such file or directory
 // CHECK-AST-PCH: <undeserialized declarations>
 // CHECK-AST-NOT: <undeserialized declarations>
 
-- 
cgit v1.2.3


From 2a41350aabd8b7d3e406141a55ce0bb6f5e70a76 Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Wed, 18 Jun 2025 17:15:12 +0100
Subject: =?UTF-8?q?Fix=20bazel=20build=20issue=20caused=20by=20#142986=20s?=
 =?UTF-8?q?econd=20attempt=20(#144721=20didnt=E2=80=A6=20(#144743)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

… cover everything)
---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel      | 1 +
 utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel | 1 +
 2 files changed, 2 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 48f2d0900d3e..761a93ea7dfa 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -12796,6 +12796,7 @@ cc_library(
         "lib/Dialect/Bufferization/IR/BufferViewFlowOpInterface.cpp",
         "lib/Dialect/Bufferization/IR/BufferizableOpInterface.cpp",
         "lib/Dialect/Bufferization/IR/BufferizationDialect.cpp",
+        "lib/Dialect/Bufferization/IR/BufferizationTypeInterfaces.cpp",
         "lib/Dialect/Bufferization/IR/BufferizationOps.cpp",
         "lib/Dialect/Bufferization/IR/UnstructuredControlFlow.cpp",
     ],
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 0eaf86da7f27..a439fdd50d21 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -375,6 +375,7 @@ cc_library(
         "//llvm:IRReader",
         "//llvm:Support",
         "//mlir:ArithDialect",
+        "//mlir:BufferizationDialect",
         "//mlir:BufferizationInterfaces",
         "//mlir:BytecodeOpInterface",
         "//mlir:CallOpInterfaces",
-- 
cgit v1.2.3


From dd40c460c42d075c47f0d1a6d83f129655eafe10 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Wed, 18 Jun 2025 12:16:01 -0400
Subject: [libc++] Clean up casts in std::forward_list (#130310)

The patch removes unnecessary casts to `void*` pointers, inline some
casts, and eliminates an identity cast.
---
 libcxx/include/forward_list | 97 ++++++++++++++++++---------------------------
 1 file changed, 39 insertions(+), 58 deletions(-)

diff --git a/libcxx/include/forward_list b/libcxx/include/forward_list
index e9b2c860b89c..bad0c11b7c7e 100644
--- a/libcxx/include/forward_list
+++ b/libcxx/include/forward_list
@@ -282,7 +282,6 @@ struct __forward_node_traits {
   typedef _NodePtr __node_pointer;
   typedef __forward_begin_node<_NodePtr> __begin_node;
   typedef __rebind_pointer_t<_NodePtr, __begin_node> __begin_node_pointer;
-  typedef __rebind_pointer_t<_NodePtr, void> __void_pointer;
 
 // TODO(LLVM 22): Remove this check
 #  ifndef _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB
@@ -294,10 +293,6 @@ struct __forward_node_traits {
                 "is being broken between LLVM 19 and LLVM 20. If you don't care about your ABI being broken, define "
                 "the _LIBCPP_ABI_FORWARD_LIST_REMOVE_NODE_POINTER_UB macro to silence this diagnostic.");
 #  endif
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI static __begin_node_pointer __as_iter_node(__node_pointer __p) {
-    return std::__static_fancy_pointer_cast<__begin_node_pointer>(__p);
-  }
 };
 
 template <class _NodePtr>
@@ -309,10 +304,6 @@ struct __forward_begin_node {
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_begin_node() : __next_(nullptr) {}
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_begin_node(pointer __n) : __next_(__n) {}
-
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __next_as_begin() const {
-    return std::__static_fancy_pointer_cast<__begin_node_pointer>(__next_);
-  }
 };
 
 template <class _Tp, class _VoidPtr>
@@ -361,15 +352,9 @@ class __forward_list_iterator {
   typedef typename __traits::__begin_node __begin_node_type;
   typedef typename __traits::__node_pointer __node_pointer;
   typedef typename __traits::__begin_node_pointer __begin_node_pointer;
-  typedef typename __traits::__void_pointer __void_pointer;
 
   __begin_node_pointer __ptr_;
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const { return __ptr_; }
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
-    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_);
-  }
-
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(nullptr_t) _NOEXCEPT
       : __ptr_(nullptr) {}
 
@@ -377,7 +362,7 @@ class __forward_list_iterator {
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__begin_node_pointer __p) _NOEXCEPT : __ptr_(__p) {}
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_iterator(__node_pointer __p) _NOEXCEPT
-      : __ptr_(__traits::__as_iter_node(__p)) {}
+      : __ptr_(std::__static_fancy_pointer_cast<__begin_node_pointer>(__p)) {}
 
   template <class, class>
   friend class forward_list;
@@ -394,14 +379,14 @@ public:
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator() _NOEXCEPT : __ptr_(nullptr) {}
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
-    return __get_unsafe_node_pointer()->__get_value();
+    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_)->__get_value();
   }
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
-    return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
+    return pointer_traits<pointer>::pointer_to(std::__static_fancy_pointer_cast<__node_pointer>(__ptr_)->__get_value());
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator& operator++() {
-    __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
+    __ptr_ = std::__static_fancy_pointer_cast<__begin_node_pointer>(__ptr_->__next_);
     return *this;
   }
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_iterator operator++(int) {
@@ -430,15 +415,9 @@ class __forward_list_const_iterator {
   typedef typename __traits::__begin_node __begin_node_type;
   typedef typename __traits::__node_pointer __node_pointer;
   typedef typename __traits::__begin_node_pointer __begin_node_pointer;
-  typedef typename __traits::__void_pointer __void_pointer;
 
   __begin_node_pointer __ptr_;
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __begin_node_pointer __get_begin() const { return __ptr_; }
-  _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __node_pointer __get_unsafe_node_pointer() const {
-    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_);
-  }
-
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(nullptr_t) _NOEXCEPT
       : __ptr_(nullptr) {}
 
@@ -447,7 +426,7 @@ class __forward_list_const_iterator {
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26
   _LIBCPP_HIDE_FROM_ABI explicit __forward_list_const_iterator(__node_pointer __p) _NOEXCEPT
-      : __ptr_(__traits::__as_iter_node(__p)) {}
+      : __ptr_(std::__static_fancy_pointer_cast<__begin_node_pointer>(__p)) {}
 
   template <class, class>
   friend class forward_list;
@@ -464,14 +443,14 @@ public:
   __forward_list_const_iterator(__forward_list_iterator<__node_pointer> __p) _NOEXCEPT : __ptr_(__p.__ptr_) {}
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI reference operator*() const {
-    return __get_unsafe_node_pointer()->__get_value();
+    return std::__static_fancy_pointer_cast<__node_pointer>(__ptr_)->__get_value();
   }
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI pointer operator->() const {
-    return pointer_traits<pointer>::pointer_to(__get_unsafe_node_pointer()->__get_value());
+    return pointer_traits<pointer>::pointer_to(std::__static_fancy_pointer_cast<__node_pointer>(__ptr_)->__get_value());
   }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator& operator++() {
-    __ptr_ = __traits::__as_iter_node(__ptr_->__next_);
+    __ptr_ = std::__static_fancy_pointer_cast<__begin_node_pointer>(__ptr_->__next_);
     return *this;
   }
   _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI __forward_list_const_iterator operator++(int) {
@@ -963,7 +942,8 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 inline forward_list<_Tp, _Alloc>::forward_list(con
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(size_type __n) {
   if (__n > 0) {
-    for (__begin_node_pointer __p = __base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
+    for (__begin_node_pointer __p = __base::__before_begin(); __n > 0;
+         --__n, __p = std::__static_fancy_pointer_cast<__begin_node_pointer>(__p->__next_)) {
       __p->__next_ = this->__create_node(/* next = */ nullptr);
     }
   }
@@ -974,7 +954,8 @@ template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 forward_list<_Tp, _Alloc>::forward_list(size_type __n, const allocator_type& __base_alloc)
     : __base(__base_alloc) {
   if (__n > 0) {
-    for (__begin_node_pointer __p = __base::__before_begin(); __n > 0; --__n, __p = __p->__next_as_begin()) {
+    for (__begin_node_pointer __p = __base::__before_begin(); __n > 0;
+         --__n, __p = std::__static_fancy_pointer_cast<__begin_node_pointer>(__p->__next_)) {
       __p->__next_ = this->__create_node(/* next = */ nullptr);
     }
   }
@@ -1167,7 +1148,7 @@ template <class _Tp, class _Alloc>
 template <class... _Args>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::emplace_after(const_iterator __p, _Args&&... __args) {
-  __begin_node_pointer const __r = __p.__get_begin();
+  __begin_node_pointer const __r = __p.__ptr_;
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::forward<_Args>(__args)...);
   return iterator(__r->__next_);
 }
@@ -1175,7 +1156,7 @@ forward_list<_Tp, _Alloc>::emplace_after(const_iterator __p, _Args&&... __args)
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
-  __begin_node_pointer const __r = __p.__get_begin();
+  __begin_node_pointer const __r = __p.__ptr_;
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, std::move(__v));
   return iterator(__r->__next_);
 }
@@ -1185,7 +1166,7 @@ forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, value_type&& __v) {
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::insert_after(const_iterator __p, const value_type& __v) {
-  __begin_node_pointer const __r = __p.__get_begin();
+  __begin_node_pointer const __r = __p.__ptr_;
   __r->__next_                   = this->__create_node(/* next = */ __r->__next_, __v);
   return iterator(__r->__next_);
 }
@@ -1194,7 +1175,7 @@ template <class _Tp, class _Alloc>
 template <class... _Args>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::__insert_after(const_iterator __p, size_type __n, _Args&&... __args) {
-  __begin_node_pointer __r = __p.__get_begin();
+  __begin_node_pointer __r = __p.__ptr_;
   if (__n > 0) {
     __node_pointer __first = this->__create_node(/* next = */ nullptr, std::forward<_Args>(__args)...);
     __node_pointer __last  = __first;
@@ -1216,7 +1197,7 @@ forward_list<_Tp, _Alloc>::__insert_after(const_iterator __p, size_type __n, _Ar
 #  endif // _LIBCPP_HAS_EXCEPTIONS
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
-    __r             = __forward_node_traits<__node_pointer>::__as_iter_node(__last);
+    __r             = std::__static_fancy_pointer_cast<__begin_node_pointer>(__last);
   }
   return iterator(__r);
 }
@@ -1232,7 +1213,7 @@ template <class _Tp, class _Alloc>
 template <class _InputIterator, class _Sentinel>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 _LIBCPP_HIDE_FROM_ABI typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _InputIterator __f, _Sentinel __l) {
-  __begin_node_pointer __r = __p.__get_begin();
+  __begin_node_pointer __r = __p.__ptr_;
 
   if (__f != __l) {
     __node_pointer __first = this->__create_node(/* next = */ nullptr, *__f);
@@ -1257,7 +1238,7 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
 
     __last->__next_ = __r->__next_;
     __r->__next_    = __first;
-    __r             = __forward_node_traits<__node_pointer>::__as_iter_node(__last);
+    __r             = std::__static_fancy_pointer_cast<__begin_node_pointer>(__last);
   }
 
   return iterator(__r);
@@ -1266,7 +1247,7 @@ forward_list<_Tp, _Alloc>::__insert_after_with_sentinel(const_iterator __p, _Inp
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::erase_after(const_iterator __f) {
-  __begin_node_pointer __p = __f.__get_begin();
+  __begin_node_pointer __p = __f.__ptr_;
   __node_pointer __n       = __p->__next_;
   __p->__next_             = __n->__next_;
   this->__delete_node(__n);
@@ -1276,9 +1257,9 @@ forward_list<_Tp, _Alloc>::erase_after(const_iterator __f) {
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 typename forward_list<_Tp, _Alloc>::iterator
 forward_list<_Tp, _Alloc>::erase_after(const_iterator __f, const_iterator __l) {
-  __node_pointer __e = __l.__get_unsafe_node_pointer();
+  __node_pointer __e = std::__static_fancy_pointer_cast<__node_pointer>(__l.__ptr_);
   if (__f != __l) {
-    __begin_node_pointer __bp = __f.__get_begin();
+    __begin_node_pointer __bp = __f.__ptr_;
 
     __node_pointer __n = __bp->__next_;
     if (__n != __e) {
@@ -1324,13 +1305,13 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::resize(size_type _
 template <class _Tp, class _Alloc>
 _LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& __x) {
   if (!__x.empty()) {
-    if (__p.__get_begin()->__next_ != nullptr) {
+    if (__p.__ptr_->__next_ != nullptr) {
       const_iterator __lm1 = __x.before_begin();
-      while (__lm1.__get_begin()->__next_ != nullptr)
+      while (__lm1.__ptr_->__next_ != nullptr)
         ++__lm1;
-      __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
+      __lm1.__ptr_->__next_ = __p.__ptr_->__next_;
     }
-    __p.__get_begin()->__next_    = __x.__before_begin()->__next_;
+    __p.__ptr_->__next_           = __x.__before_begin()->__next_;
     __x.__before_begin()->__next_ = nullptr;
   }
 }
@@ -1340,9 +1321,9 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void
 forward_list<_Tp, _Alloc>::splice_after(const_iterator __p, forward_list& /*__other*/, const_iterator __i) {
   const_iterator __lm1 = std::next(__i);
   if (__p != __i && __p != __lm1) {
-    __i.__get_begin()->__next_   = __lm1.__get_begin()->__next_;
-    __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
-    __p.__get_begin()->__next_   = __lm1.__get_unsafe_node_pointer();
+    __i.__ptr_->__next_   = __lm1.__ptr_->__next_;
+    __lm1.__ptr_->__next_ = __p.__ptr_->__next_;
+    __p.__ptr_->__next_   = std::__static_fancy_pointer_cast<__node_pointer>(__lm1.__ptr_);
   }
 }
 
@@ -1351,12 +1332,12 @@ _LIBCPP_CONSTEXPR_SINCE_CXX26 void forward_list<_Tp, _Alloc>::splice_after(
     const_iterator __p, forward_list& /*__other*/, const_iterator __f, const_iterator __l) {
   if (__f != __l && __p != __f) {
     const_iterator __lm1 = __f;
-    while (__lm1.__get_begin()->__next_ != __l.__get_begin())
+    while (__lm1.__ptr_->__next_ != __l.__ptr_)
       ++__lm1;
     if (__f != __lm1) {
-      __lm1.__get_begin()->__next_ = __p.__get_begin()->__next_;
-      __p.__get_begin()->__next_   = __f.__get_begin()->__next_;
-      __f.__get_begin()->__next_   = __l.__get_unsafe_node_pointer();
+      __lm1.__ptr_->__next_ = __p.__ptr_->__next_;
+      __p.__ptr_->__next_   = __f.__ptr_->__next_;
+      __f.__ptr_->__next_   = std::__static_fancy_pointer_cast<__node_pointer>(__l.__ptr_);
     }
   }
 }
@@ -1385,8 +1366,8 @@ forward_list<_Tp, _Alloc>::remove(const value_type& __v) {
   forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
   const iterator __e                                            = end();
-  for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;) {
-    if (__i.__get_begin()->__next_->__get_value() == __v) {
+  for (iterator __i = before_begin(); __i.__ptr_->__next_ != nullptr;) {
+    if (__i.__ptr_->__next_->__get_value() == __v) {
       ++__count_removed;
       iterator __j = std::next(__i, 2);
       for (; __j != __e && *__j == __v; ++__j)
@@ -1409,8 +1390,8 @@ forward_list<_Tp, _Alloc>::remove_if(_Predicate __pred) {
   forward_list<_Tp, _Alloc> __deleted_nodes(get_allocator()); // collect the nodes we're removing
   typename forward_list<_Tp, _Alloc>::size_type __count_removed = 0;
   const iterator __e                                            = end();
-  for (iterator __i = before_begin(); __i.__get_begin()->__next_ != nullptr;) {
-    if (__pred(__i.__get_begin()->__next_->__get_value())) {
+  for (iterator __i = before_begin(); __i.__ptr_->__next_ != nullptr;) {
+    if (__pred(__i.__ptr_->__next_->__get_value())) {
       ++__count_removed;
       iterator __j = std::next(__i, 2);
       for (; __j != __e && __pred(*__j); ++__j)
@@ -1436,7 +1417,7 @@ forward_list<_Tp, _Alloc>::unique(_BinaryPredicate __binary_pred) {
     iterator __j = std::next(__i);
     for (; __j != __e && __binary_pred(*__i, *__j); ++__j)
       ++__count_removed;
-    if (__i.__get_begin()->__next_ != __j.__get_unsafe_node_pointer())
+    if (__i.__ptr_->__next_ != std::__static_fancy_pointer_cast<__node_pointer>(__j.__ptr_))
       __deleted_nodes.splice_after(__deleted_nodes.before_begin(), *this, __i, __j);
     __i = __j;
   }
@@ -1516,7 +1497,7 @@ forward_list<_Tp, _Alloc>::__sort(__node_pointer __f1, difference_type __sz, _Co
   }
   difference_type __sz1 = __sz / 2;
   difference_type __sz2 = __sz - __sz1;
-  __node_pointer __t    = std::next(iterator(__f1), __sz1 - 1).__get_unsafe_node_pointer();
+  __node_pointer __t    = std::__static_fancy_pointer_cast<__node_pointer>(std::next(iterator(__f1), __sz1 - 1).__ptr_);
   __node_pointer __f2   = __t->__next_;
   __t->__next_          = nullptr;
   return __merge(__sort(__f1, __sz1, __comp), __sort(__f2, __sz2, __comp), __comp);
-- 
cgit v1.2.3


From 9827440f1e723423baf4c235e844eb8ac48a8f97 Mon Sep 17 00:00:00 2001
From: Peng Liu <winner245@hotmail.com>
Date: Wed, 18 Jun 2025 12:22:47 -0400
Subject: [libc++] Optimize ranges::{for_each, for_each_n} for segmented
 iterators (#132896)

Previously, the segmented iterator optimization was limited to `std::{for_each, for_each_n}`. This patch
extends the optimization to `std::ranges::for_each` and `std::ranges::for_each_n`, ensuring consistent
optimizations across these algorithms. This patch first generalizes the `std` algorithms by introducing
a `Projection` parameter, which is set to `__identity` for the `std` algorithms. Then we let the `ranges`
algorithms to directly call their `std` counterparts with a general `__proj` argument. Benchmarks
demonstrate performance improvements of up to 21.4x for ``std::deque::iterator`` and 22.3x for
``join_view`` of ``vector<vector<char>>``.

Addresses a subtask of #102817.
---
 libcxx/docs/ReleaseNotes/21.rst                    |  5 ++-
 libcxx/include/__algorithm/for_each.h              | 35 ++++++++++------
 libcxx/include/__algorithm/for_each_n.h            | 26 +++++++-----
 libcxx/include/__algorithm/ranges_for_each.h       | 18 +++++++--
 libcxx/include/__algorithm/ranges_for_each_n.h     |  9 ++---
 libcxx/include/experimental/iterator               |  1 +
 libcxx/include/mutex                               |  1 +
 libcxx/include/shared_mutex                        |  1 +
 .../algorithms/nonmodifying/for_each.bench.cpp     | 43 ++++++++++++++++++--
 .../algorithms/nonmodifying/for_each_n.bench.cpp   | 23 +++++------
 .../alg.foreach/ranges.for_each.pass.cpp           | 46 +++++++++++++++++++---
 .../alg.foreach/ranges.for_each_n.pass.cpp         | 46 +++++++++++++++++++++-
 12 files changed, 197 insertions(+), 57 deletions(-)

diff --git a/libcxx/docs/ReleaseNotes/21.rst b/libcxx/docs/ReleaseNotes/21.rst
index 2a5b90750eaf..8661e5898fbc 100644
--- a/libcxx/docs/ReleaseNotes/21.rst
+++ b/libcxx/docs/ReleaseNotes/21.rst
@@ -70,8 +70,9 @@ Improvements and New Features
 - The segmented iterator optimization for ``std::for_each`` has been backported to C++11. Previously it was only available
   in C++23 and later.
 
-- The ``std::for_each_n`` algorithm has been optimized for segmented iterators, resulting in a performance improvement of
-  up to 17.7x for ``std::deque<short>`` iterators, and up to 13.9x for ``std::join_view<vector<vector<short>>>`` iterators.
+- The ``std::for_each_n``, ``std::ranges::for_each`` and ``std::ranges::for_each_n`` algorithms have been optimized for
+  segmented iterators, resulting in a performance improvement of up to 17.7x for ``std::deque<short>`` iterators, and up
+  to 13.9x for ``std::join_view<vector<vector<short>>>`` iterators.
 
 - The ``bitset::to_string`` function has been optimized, resulting in a performance improvement of up to 8.3x for bitsets
   with uniformly distributed zeros and ones, and up to 13.5x and 16.1x for sparse and dense bitsets, respectively.
diff --git a/libcxx/include/__algorithm/for_each.h b/libcxx/include/__algorithm/for_each.h
index b6c2c7c056ed..4167eec3506e 100644
--- a/libcxx/include/__algorithm/for_each.h
+++ b/libcxx/include/__algorithm/for_each.h
@@ -12,41 +12,54 @@
 
 #include <__algorithm/for_each_segment.h>
 #include <__config>
+#include <__functional/identity.h>
 #include <__iterator/segmented_iterator.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/invoke.h>
+#include <__utility/move.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
 #  pragma GCC system_header
 #endif
 
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _InputIterator, class _Sent, class _Func>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __for_each(_InputIterator __first, _Sent __last, _Func& __f) {
+template <class _InputIterator, class _Sent, class _Func, class _Proj>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
+__for_each(_InputIterator __first, _Sent __last, _Func& __f, _Proj& __proj) {
   for (; __first != __last; ++__first)
-    __f(*__first);
+    std::__invoke(__f, std::__invoke(__proj, *__first));
+  return __first;
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 template <class _SegmentedIterator,
-          class _Function,
+          class _Func,
+          class _Proj,
           __enable_if_t<__is_segmented_iterator<_SegmentedIterator>::value, int> = 0>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void
-__for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Function& __func) {
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _SegmentedIterator
+__for_each(_SegmentedIterator __first, _SegmentedIterator __last, _Func& __func, _Proj& __proj) {
   using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
   std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
-    std::__for_each(__lfirst, __llast, __func);
+    std::__for_each(__lfirst, __llast, __func, __proj);
   });
+  return __last;
 }
 #endif // !_LIBCPP_CXX03_LANG
 
-template <class _InputIterator, class _Function>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Function
-for_each(_InputIterator __first, _InputIterator __last, _Function __f) {
-  std::__for_each(__first, __last, __f);
+template <class _InputIterator, class _Func>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Func
+for_each(_InputIterator __first, _InputIterator __last, _Func __f) {
+  __identity __proj;
+  std::__for_each(__first, __last, __f, __proj);
   return __f;
 }
 
 _LIBCPP_END_NAMESPACE_STD
 
+_LIBCPP_POP_MACROS
+
 #endif // _LIBCPP___ALGORITHM_FOR_EACH_H
diff --git a/libcxx/include/__algorithm/for_each_n.h b/libcxx/include/__algorithm/for_each_n.h
index 29351ec39f4e..9a6c6bb5175d 100644
--- a/libcxx/include/__algorithm/for_each_n.h
+++ b/libcxx/include/__algorithm/for_each_n.h
@@ -13,10 +13,12 @@
 #include <__algorithm/for_each.h>
 #include <__algorithm/for_each_n_segment.h>
 #include <__config>
+#include <__functional/identity.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
 #include <__type_traits/disjunction.h>
 #include <__type_traits/enable_if.h>
+#include <__type_traits/invoke.h>
 #include <__type_traits/negation.h>
 #include <__utility/convert_to_integral.h>
 #include <__utility/move.h>
@@ -33,16 +35,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _InputIterator,
           class _Size,
           class _Func,
+          class _Proj,
           __enable_if_t<!__has_random_access_iterator_category<_InputIterator>::value &&
                             _Or< _Not<__is_segmented_iterator<_InputIterator> >,
                                  _Not<__has_random_access_local_iterator<_InputIterator> > >::value,
                         int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
-__for_each_n(_InputIterator __first, _Size __orig_n, _Func& __f) {
+__for_each_n(_InputIterator __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
   typedef decltype(std::__convert_to_integral(__orig_n)) _IntegralSize;
   _IntegralSize __n = __orig_n;
   while (__n > 0) {
-    __f(*__first);
+    std::__invoke(__f, std::__invoke(__proj, *__first));
     ++__first;
     --__n;
   }
@@ -52,39 +55,42 @@ __for_each_n(_InputIterator __first, _Size __orig_n, _Func& __f) {
 template <class _RandIter,
           class _Size,
           class _Func,
+          class _Proj,
           __enable_if_t<__has_random_access_iterator_category<_RandIter>::value, int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _RandIter
-__for_each_n(_RandIter __first, _Size __orig_n, _Func& __f) {
+__for_each_n(_RandIter __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
   typename std::iterator_traits<_RandIter>::difference_type __n = __orig_n;
   auto __last                                                   = __first + __n;
-  std::__for_each(__first, __last, __f);
-  return std::move(__last);
+  std::__for_each(__first, __last, __f, __proj);
+  return __last;
 }
 
 #ifndef _LIBCPP_CXX03_LANG
 template <class _SegmentedIterator,
           class _Size,
           class _Func,
+          class _Proj,
           __enable_if_t<!__has_random_access_iterator_category<_SegmentedIterator>::value &&
                             __is_segmented_iterator<_SegmentedIterator>::value &&
                             __has_random_access_iterator_category<
                                 typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator>::value,
                         int> = 0>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _SegmentedIterator
-__for_each_n(_SegmentedIterator __first, _Size __orig_n, _Func& __f) {
+__for_each_n(_SegmentedIterator __first, _Size __orig_n, _Func& __f, _Proj& __proj) {
   using __local_iterator_t = typename __segmented_iterator_traits<_SegmentedIterator>::__local_iterator;
   return std::__for_each_n_segment(__first, __orig_n, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
-    std::__for_each(__lfirst, __llast, __f);
+    std::__for_each(__lfirst, __llast, __f, __proj);
   });
 }
 #endif // !_LIBCPP_CXX03_LANG
 
 #if _LIBCPP_STD_VER >= 17
 
-template <class _InputIterator, class _Size, class _Function>
+template <class _InputIterator, class _Size, class _Func>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
-for_each_n(_InputIterator __first, _Size __orig_n, _Function __f) {
-  return std::__for_each_n(__first, __orig_n, __f);
+for_each_n(_InputIterator __first, _Size __orig_n, _Func __f) {
+  __identity __proj;
+  return std::__for_each_n(__first, __orig_n, __f, __proj);
 }
 
 #endif // _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/__algorithm/ranges_for_each.h b/libcxx/include/__algorithm/ranges_for_each.h
index de39bc552275..e9c84e8583f8 100644
--- a/libcxx/include/__algorithm/ranges_for_each.h
+++ b/libcxx/include/__algorithm/ranges_for_each.h
@@ -9,10 +9,12 @@
 #ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
 #define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_H
 
+#include <__algorithm/for_each.h>
+#include <__algorithm/for_each_n.h>
 #include <__algorithm/in_fun_result.h>
+#include <__concepts/assignable.h>
 #include <__config>
 #include <__functional/identity.h>
-#include <__functional/invoke.h>
 #include <__iterator/concepts.h>
 #include <__iterator/projected.h>
 #include <__ranges/access.h>
@@ -41,9 +43,17 @@ private:
   template <class _Iter, class _Sent, class _Proj, class _Func>
   _LIBCPP_HIDE_FROM_ABI constexpr static for_each_result<_Iter, _Func>
   __for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
-    for (; __first != __last; ++__first)
-      std::invoke(__func, std::invoke(__proj, *__first));
-    return {std::move(__first), std::move(__func)};
+    // In the case where we have different iterator and sentinel types, the segmented iterator optimization
+    // in std::for_each will not kick in. Therefore, we prefer std::for_each_n in that case (whenever we can
+    // obtain the `n`).
+    if constexpr (!std::assignable_from<_Iter&, _Sent> && std::sized_sentinel_for<_Sent, _Iter>) {
+      auto __n   = __last - __first;
+      auto __end = std::__for_each_n(std::move(__first), __n, __func, __proj);
+      return {std::move(__end), std::move(__func)};
+    } else {
+      auto __end = std::__for_each(std::move(__first), std::move(__last), __func, __proj);
+      return {std::move(__end), std::move(__func)};
+    }
   }
 
 public:
diff --git a/libcxx/include/__algorithm/ranges_for_each_n.h b/libcxx/include/__algorithm/ranges_for_each_n.h
index 603cb723233c..3aab1b79c10a 100644
--- a/libcxx/include/__algorithm/ranges_for_each_n.h
+++ b/libcxx/include/__algorithm/ranges_for_each_n.h
@@ -9,10 +9,10 @@
 #ifndef _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
 #define _LIBCPP___ALGORITHM_RANGES_FOR_EACH_N_H
 
+#include <__algorithm/for_each_n.h>
 #include <__algorithm/in_fun_result.h>
 #include <__config>
 #include <__functional/identity.h>
-#include <__functional/invoke.h>
 #include <__iterator/concepts.h>
 #include <__iterator/incrementable_traits.h>
 #include <__iterator/iterator_traits.h>
@@ -40,11 +40,8 @@ struct __for_each_n {
   template <input_iterator _Iter, class _Proj = identity, indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
   _LIBCPP_HIDE_FROM_ABI constexpr for_each_n_result<_Iter, _Func>
   operator()(_Iter __first, iter_difference_t<_Iter> __count, _Func __func, _Proj __proj = {}) const {
-    while (__count-- > 0) {
-      std::invoke(__func, std::invoke(__proj, *__first));
-      ++__first;
-    }
-    return {std::move(__first), std::move(__func)};
+    auto __last = std::__for_each_n(std::move(__first), __count, __func, __proj);
+    return {std::move(__last), std::move(__func)};
   }
 };
 
diff --git a/libcxx/include/experimental/iterator b/libcxx/include/experimental/iterator
index d92613845a66..565bb83903ac 100644
--- a/libcxx/include/experimental/iterator
+++ b/libcxx/include/experimental/iterator
@@ -127,6 +127,7 @@ _LIBCPP_POP_MACROS
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
 #    include <cstddef>
 #    include <iosfwd>
+#    include <optional>
 #    include <type_traits>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
diff --git a/libcxx/include/mutex b/libcxx/include/mutex
index e058b3113073..f616bad3ac17 100644
--- a/libcxx/include/mutex
+++ b/libcxx/include/mutex
@@ -504,6 +504,7 @@ _LIBCPP_POP_MACROS
 #    include <initializer_list>
 #    include <iosfwd>
 #    include <new>
+#    include <optional>
 #    include <stdexcept>
 #    include <system_error>
 #    include <type_traits>
diff --git a/libcxx/include/shared_mutex b/libcxx/include/shared_mutex
index e6759e413dfe..6469c02ca587 100644
--- a/libcxx/include/shared_mutex
+++ b/libcxx/include/shared_mutex
@@ -457,6 +457,7 @@ _LIBCPP_POP_MACROS
 #  endif // _LIBCPP_HAS_THREADS
 
 #  if !defined(_LIBCPP_REMOVE_TRANSITIVE_INCLUDES) && _LIBCPP_STD_VER <= 20
+#    include <optional>
 #    include <system_error>
 #  endif
 #endif // __cplusplus < 201103L && defined(_LIBCPP_USE_FROZEN_CXX03_HEADERS)
diff --git a/libcxx/test/benchmarks/algorithms/nonmodifying/for_each.bench.cpp b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each.bench.cpp
index 760accbe4d92..f58f336f8b89 100644
--- a/libcxx/test/benchmarks/algorithms/nonmodifying/for_each.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each.bench.cpp
@@ -12,6 +12,7 @@
 #include <cstddef>
 #include <deque>
 #include <list>
+#include <ranges>
 #include <string>
 #include <vector>
 
@@ -23,6 +24,7 @@ int main(int argc, char** argv) {
   // {std,ranges}::for_each
   {
     auto bm = []<class Container>(std::string name, auto for_each) {
+      using ElemType = typename Container::value_type;
       benchmark::RegisterBenchmark(
           name,
           [for_each](auto& st) {
@@ -33,15 +35,14 @@ int main(int argc, char** argv) {
 
             for ([[maybe_unused]] auto _ : st) {
               benchmark::DoNotOptimize(c);
-              auto result = for_each(first, last, [](int& x) { x = std::clamp(x, 10, 100); });
+              auto result = for_each(first, last, [](ElemType& x) { x = std::clamp<ElemType>(x, 10, 100); });
               benchmark::DoNotOptimize(result);
             }
           })
           ->Arg(8)
           ->Arg(32)
           ->Arg(50) // non power-of-two
-          ->Arg(8192)
-          ->Arg(1 << 20);
+          ->Arg(8192);
     };
     bm.operator()<std::vector<int>>("std::for_each(vector<int>)", std_for_each);
     bm.operator()<std::deque<int>>("std::for_each(deque<int>)", std_for_each);
@@ -51,6 +52,42 @@ int main(int argc, char** argv) {
     bm.operator()<std::list<int>>("rng::for_each(list<int>)", std::ranges::for_each);
   }
 
+  // {std,ranges}::for_each for join_view
+  {
+    auto bm = []<class Container>(std::string name, auto for_each) {
+      using C1       = typename Container::value_type;
+      using ElemType = typename C1::value_type;
+
+      benchmark::RegisterBenchmark(
+          name,
+          [for_each](auto& st) {
+            std::size_t const size     = st.range(0);
+            std::size_t const seg_size = 256;
+            std::size_t const segments = (size + seg_size - 1) / seg_size;
+            Container c(segments);
+            for (std::size_t i = 0, n = size; i < segments; ++i, n -= seg_size) {
+              c[i].resize(std::min(seg_size, n), ElemType(1));
+            }
+
+            auto view  = c | std::views::join;
+            auto first = view.begin();
+            auto last  = view.end();
+
+            for ([[maybe_unused]] auto _ : st) {
+              benchmark::DoNotOptimize(c);
+              auto result = for_each(first, last, [](ElemType& x) { x = std::clamp<ElemType>(x, 10, 100); });
+              benchmark::DoNotOptimize(result);
+            }
+          })
+          ->Arg(8)
+          ->Arg(32)
+          ->Arg(50) // non power-of-two
+          ->Arg(8192);
+    };
+    bm.operator()<std::vector<std::vector<int>>>("std::for_each(join_view(vector<vector<int>>))", std_for_each);
+    bm.operator()<std::vector<std::vector<int>>>("rng::for_each(join_view(vector<vector<int>>)", std::ranges::for_each);
+  }
+
   benchmark::Initialize(&argc, argv);
   benchmark::RunSpecifiedBenchmarks();
   benchmark::Shutdown();
diff --git a/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
index 784708c7e01e..e643e647722c 100644
--- a/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
+++ b/libcxx/test/benchmarks/algorithms/nonmodifying/for_each_n.bench.cpp
@@ -21,7 +21,7 @@
 int main(int argc, char** argv) {
   auto std_for_each_n = [](auto first, auto n, auto f) { return std::for_each_n(first, n, f); };
 
-  // std::for_each_n
+  // {std,ranges}::for_each_n
   {
     auto bm = []<class Container>(std::string name, auto for_each_n) {
       using ElemType = typename Container::value_type;
@@ -41,19 +41,17 @@ int main(int argc, char** argv) {
           ->Arg(8)
           ->Arg(32)
           ->Arg(50) // non power-of-two
-          ->Arg(1024)
-          ->Arg(4096)
-          ->Arg(8192)
-          ->Arg(1 << 14)
-          ->Arg(1 << 16)
-          ->Arg(1 << 18);
+          ->Arg(8192);
     };
     bm.operator()<std::vector<int>>("std::for_each_n(vector<int>)", std_for_each_n);
     bm.operator()<std::deque<int>>("std::for_each_n(deque<int>)", std_for_each_n);
     bm.operator()<std::list<int>>("std::for_each_n(list<int>)", std_for_each_n);
+    bm.operator()<std::vector<int>>("rng::for_each_n(vector<int>)", std::ranges::for_each_n);
+    bm.operator()<std::deque<int>>("rng::for_each_n(deque<int>)", std::ranges::for_each_n);
+    bm.operator()<std::list<int>>("rng::for_each_n(list<int>)", std::ranges::for_each_n);
   }
 
-  // std::for_each_n for join_view
+  // {std,ranges}::for_each_n for join_view
   {
     auto bm = []<class Container>(std::string name, auto for_each_n) {
       using C1       = typename Container::value_type;
@@ -81,14 +79,11 @@ int main(int argc, char** argv) {
           ->Arg(8)
           ->Arg(32)
           ->Arg(50) // non power-of-two
-          ->Arg(1024)
-          ->Arg(4096)
-          ->Arg(8192)
-          ->Arg(1 << 14)
-          ->Arg(1 << 16)
-          ->Arg(1 << 18);
+          ->Arg(8192);
     };
     bm.operator()<std::vector<std::vector<int>>>("std::for_each_n(join_view(vector<vector<int>>))", std_for_each_n);
+    bm.operator()<std::vector<std::vector<int>>>(
+        "rng::for_each_n(join_view(vector<vector<int>>)", std::ranges::for_each_n);
   }
 
   benchmark::Initialize(&argc, argv);
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
index 8b9b6e82cbcb..a6d0afde3186 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each.pass.cpp
@@ -20,7 +20,10 @@
 
 #include <algorithm>
 #include <array>
+#include <cassert>
+#include <deque>
 #include <ranges>
+#include <vector>
 
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
@@ -30,7 +33,7 @@ struct Callable {
 };
 
 template <class Iter, class Sent = Iter>
-concept HasForEachIt = requires (Iter iter, Sent sent) { std::ranges::for_each(iter, sent, Callable{}); };
+concept HasForEachIt = requires(Iter iter, Sent sent) { std::ranges::for_each(iter, sent, Callable{}); };
 
 static_assert(HasForEachIt<int*>);
 static_assert(!HasForEachIt<InputIteratorNotDerivedFrom>);
@@ -47,7 +50,7 @@ static_assert(!HasForEachItFunc<IndirectUnaryPredicateNotPredicate>);
 static_assert(!HasForEachItFunc<IndirectUnaryPredicateNotCopyConstructible>);
 
 template <class Range>
-concept HasForEachR = requires (Range range) { std::ranges::for_each(range, Callable{}); };
+concept HasForEachR = requires(Range range) { std::ranges::for_each(range, Callable{}); };
 
 static_assert(HasForEachR<UncheckedRange<int*>>);
 static_assert(!HasForEachR<InputRangeNotDerivedFrom>);
@@ -68,7 +71,7 @@ constexpr void test_iterator() {
   { // simple test
     {
       auto func = [i = 0](int& a) mutable { a += i++; };
-      int a[] = {1, 6, 3, 4};
+      int a[]   = {1, 6, 3, 4};
       std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> decltype(auto) ret =
           std::ranges::for_each(Iter(a), Sent(Iter(a + 4)), func);
       assert(a[0] == 1);
@@ -81,8 +84,8 @@ constexpr void test_iterator() {
       assert(i == 4);
     }
     {
-      auto func = [i = 0](int& a) mutable { a += i++; };
-      int a[] = {1, 6, 3, 4};
+      auto func  = [i = 0](int& a) mutable { a += i++; };
+      int a[]    = {1, 6, 3, 4};
       auto range = std::ranges::subrange(Iter(a), Sent(Iter(a + 4)));
       std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> decltype(auto) ret =
           std::ranges::for_each(range, func);
@@ -110,6 +113,30 @@ constexpr void test_iterator() {
   }
 }
 
+struct deque_test {
+  std::deque<int>* d_;
+  int* i_;
+
+  deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
+
+  void operator()(int& v) {
+    assert(&(*d_)[*i_] == &v);
+    ++*i_;
+  }
+};
+
+/*TEST_CONSTEXPR_CXX26*/
+void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX26 once std::deque is constexpr
+  // check that segmented deque iterators work properly
+  int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
+  for (const int size : sizes) {
+    std::deque<int> d(size);
+    int index = 0;
+
+    std::ranges::for_each(d, deque_test(d, index));
+  }
+}
+
 constexpr bool test() {
   test_iterator<cpp17_input_iterator<int*>, sentinel_wrapper<cpp17_input_iterator<int*>>>();
   test_iterator<cpp20_input_iterator<int*>, sentinel_wrapper<cpp20_input_iterator<int*>>>();
@@ -146,6 +173,15 @@ constexpr bool test() {
     }
   }
 
+  if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_26_OR_RUNTIME_EVALUATED when std::deque is made constexpr
+    test_segmented_deque_iterator();
+
+  {
+    std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
+    auto v                            = vec | std::views::join;
+    std::ranges::for_each(v, [i = 0](int x) mutable { assert(x == 2 * i++); }, [](int x) { return 2 * x; });
+  }
+
   return true;
 }
 
diff --git a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
index d4b2d053d08c..157876369423 100644
--- a/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
+++ b/libcxx/test/std/algorithms/alg.nonmodifying/alg.foreach/ranges.for_each_n.pass.cpp
@@ -17,7 +17,12 @@
 
 #include <algorithm>
 #include <array>
+#include <cassert>
+#include <deque>
+#include <iterator>
 #include <ranges>
+#include <ranges>
+#include <vector>
 
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
@@ -27,7 +32,7 @@ struct Callable {
 };
 
 template <class Iter>
-concept HasForEachN = requires (Iter iter) { std::ranges::for_each_n(iter, 0, Callable{}); };
+concept HasForEachN = requires(Iter iter) { std::ranges::for_each_n(iter, 0, Callable{}); };
 
 static_assert(HasForEachN<int*>);
 static_assert(!HasForEachN<InputIteratorNotDerivedFrom>);
@@ -45,7 +50,7 @@ template <class Iter>
 constexpr void test_iterator() {
   { // simple test
     auto func = [i = 0](int& a) mutable { a += i++; };
-    int a[] = {1, 6, 3, 4};
+    int a[]   = {1, 6, 3, 4};
     std::same_as<std::ranges::for_each_result<Iter, decltype(func)>> auto ret =
         std::ranges::for_each_n(Iter(a), 4, func);
     assert(a[0] == 1);
@@ -64,6 +69,30 @@ constexpr void test_iterator() {
   }
 }
 
+struct deque_test {
+  std::deque<int>* d_;
+  int* i_;
+
+  deque_test(std::deque<int>& d, int& i) : d_(&d), i_(&i) {}
+
+  void operator()(int& v) {
+    assert(&(*d_)[*i_] == &v);
+    ++*i_;
+  }
+};
+
+/*TEST_CONSTEXPR_CXX26*/
+void test_segmented_deque_iterator() { // TODO: Mark as TEST_CONSTEXPR_CXX26 once std::deque is constexpr
+  // check that segmented deque iterators work properly
+  int sizes[] = {0, 1, 2, 1023, 1024, 1025, 2047, 2048, 2049};
+  for (const int size : sizes) {
+    std::deque<int> d(size);
+    int index = 0;
+
+    std::ranges::for_each_n(d.begin(), d.size(), deque_test(d, index));
+  }
+}
+
 constexpr bool test() {
   test_iterator<cpp17_input_iterator<int*>>();
   test_iterator<cpp20_input_iterator<int*>>();
@@ -89,6 +118,19 @@ constexpr bool test() {
     assert(a[2].other == 6);
   }
 
+  if (!TEST_IS_CONSTANT_EVALUATED) // TODO: Use TEST_STD_AT_LEAST_26_OR_RUNTIME_EVALUATED when std::deque is made constexpr
+    test_segmented_deque_iterator();
+
+  {
+    std::vector<std::vector<int>> vec = {{0}, {1, 2}, {3, 4, 5}, {6, 7, 8, 9}, {10}, {11, 12, 13}};
+    auto v                            = vec | std::views::join;
+    std::ranges::for_each_n(
+        v.begin(),
+        std::ranges::distance(v),
+        [i = 0](int x) mutable { assert(x == 2 * i++); },
+        [](int x) { return 2 * x; });
+  }
+
   return true;
 }
 
-- 
cgit v1.2.3


From 00189211486d052b25429f11790ef5486cf9d3ce Mon Sep 17 00:00:00 2001
From: woruyu <99597449+woruyu@users.noreply.github.com>
Date: Thu, 19 Jun 2025 00:22:53 +0800
Subject: [DAG] add (~a | x) & (a | y) -> (a & (x ^ y)) ^y for foldMaskedMerge
 (#144342)

### Summary
This PR resolves https://github.com/llvm/llvm-project/issues/143864

Add (~a | x) & (a | y) -> (a & (x ^ y)) ^y for foldMaskedMerge func
using SDPatternMatch

aftering adding this pattern, run ```ninja check-llvm-codegen```, all
other cases remain unchanged, so I add a
testcase(fold-masked-merge-demorgan.ll) for it

---------

Co-authored-by: Simon Pilgrim <llvm-dev@redking.me.uk>
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp      |  24 +-
 .../test/CodeGen/X86/fold-masked-merge-demorgan.ll | 267 +++++++++++++++++++++
 2 files changed, 284 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/X86/fold-masked-merge-demorgan.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 934199e414c7..0e078f9dd88b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7206,24 +7206,30 @@ static SDValue foldLogicTreeOfShifts(SDNode *N, SDValue LeftHand,
   return DAG.getNode(LogicOpcode, DL, VT, CombinedShifts, W);
 }
 
-/// Fold "masked merge" expressions like `(m & x) | (~m & y)` into the
-/// equivalent `((x ^ y) & m) ^ y)` pattern.
-/// This is typically a better representation for targets without a fused
-/// "and-not" operation.
+/// Fold "masked merge" expressions like `(m & x) | (~m & y)` and its DeMorgan
+/// variant `(~m | x) & (m | y)` into the equivalent `((x ^ y) & m) ^ y)`
+/// pattern. This is typically a better representation for targets without a
+/// fused "and-not" operation.
 static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG,
                                const TargetLowering &TLI, const SDLoc &DL) {
   // Note that masked-merge variants using XOR or ADD expressions are
-  // normalized to OR by InstCombine so we only check for OR.
-  assert(Node->getOpcode() == ISD::OR && "Must be called with ISD::OR node");
+  // normalized to OR by InstCombine so we only check for OR or AND.
+  assert(Node->getOpcode() == ISD::OR ||
+         Node->getOpcode() == ISD::AND &&
+             "Must be called with ISD::OR or ISD::AND node");
 
   // If the target supports and-not, don't fold this.
   if (TLI.hasAndNot(SDValue(Node, 0)))
     return SDValue();
 
   SDValue M, X, Y;
+
   if (sd_match(Node,
                m_Or(m_OneUse(m_And(m_OneUse(m_Not(m_Value(M))), m_Value(Y))),
-                    m_OneUse(m_And(m_Deferred(M), m_Value(X)))))) {
+                    m_OneUse(m_And(m_Deferred(M), m_Value(X))))) ||
+      sd_match(Node,
+               m_And(m_OneUse(m_Or(m_OneUse(m_Not(m_Value(M))), m_Value(X))),
+                     m_OneUse(m_Or(m_Deferred(M), m_Value(Y)))))) {
     EVT VT = M.getValueType();
     SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, X, Y);
     SDValue And = DAG.getNode(ISD::AND, DL, VT, Xor, M);
@@ -7678,6 +7684,10 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
     if (SDValue R = foldLogicTreeOfShifts(N, N0, N1, DAG))
       return R;
 
+  if (VT.isScalarInteger() && VT != MVT::i1)
+    if (SDValue R = foldMaskedMerge(N, DAG, TLI, DL))
+      return R;
+
   return SDValue();
 }
 
diff --git a/llvm/test/CodeGen/X86/fold-masked-merge-demorgan.ll b/llvm/test/CodeGen/X86/fold-masked-merge-demorgan.ll
new file mode 100644
index 000000000000..fe27b3c73be0
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fold-masked-merge-demorgan.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -o - %s -mtriple=x86_64-- | FileCheck %s --check-prefixes=CHECK,NOBMI
+; RUN: llc -o - %s -mtriple=x86_64-- -mattr=+bmi | FileCheck %s --check-prefixes=CHECK,BMI
+;
+; test that masked-merge code is generated as "xor;and;xor" sequence or
+; "andn ; and; or" if and-not is available.
+
+define i32 @masked_merge0_demorgan(i32 %a0, i32 %a1, i32 %a2) {
+; NOBMI-LABEL: masked_merge0_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
+; NOBMI-NEXT:    andl %edi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: masked_merge0_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl %edi, %edx
+; BMI-NEXT:    andnl %edi, %esi, %eax
+; BMI-NEXT:    andnl %edx, %eax, %eax
+; BMI-NEXT:    retq
+  %not = xor i32 %a0, -1
+  %or0 = or i32 %not, %a1
+  %or1 = or i32 %a0, %a2
+  %and = and i32 %or0, %or1
+  ret i32 %and
+}
+
+define i16 @masked_merge1_demorgan(i16 %a0, i16 %a1, i16 %a2) {
+; NOBMI-LABEL: masked_merge1_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
+; NOBMI-NEXT:    andl %edi, %eax
+; NOBMI-NEXT:    xorl %edx, %eax
+; NOBMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: masked_merge1_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    andnl %edx, %edi, %eax
+; BMI-NEXT:    andl %edi, %esi
+; BMI-NEXT:    orl %esi, %eax
+; BMI-NEXT:    # kill: def $ax killed $ax killed $eax
+; BMI-NEXT:    retq
+  %not = xor i16 %a0, -1
+  %or0 = or i16 %not, %a1
+  %or1 = or i16 %a0, %a2
+  %and = and i16 %or0, %or1
+  ret i16 %and
+}
+
+define i8 @masked_merge2_demorgan(i8 %a0, i8 %a1, i8 %a2) {
+; CHECK-LABEL: masked_merge2_demorgan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %esi, %eax
+; CHECK-NEXT:    # kill: def $al killed $al killed $eax
+; CHECK-NEXT:    retq
+  %not = xor i8 %a0, -1
+  %or0 = or i8 %not, %a1
+  %or1 = or i8 %a0, %a1
+  %and = and i8 %or0, %or1
+  ret i8 %and
+}
+
+define i64 @masked_merge3_demorgan(i64 %a0, i64 %a1, i64 %a2) {
+; NOBMI-LABEL: masked_merge3_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movq %rsi, %rax
+; NOBMI-NEXT:    notq %rdx
+; NOBMI-NEXT:    xorq %rdx, %rax
+; NOBMI-NEXT:    notq %rax
+; NOBMI-NEXT:    andq %rdi, %rax
+; NOBMI-NEXT:    xorq %rdx, %rax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: masked_merge3_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    andnq %rdx, %rdi, %rax
+; BMI-NEXT:    andq %rdi, %rsi
+; BMI-NEXT:    notq %rsi
+; BMI-NEXT:    andnq %rsi, %rax, %rax
+; BMI-NEXT:    retq
+  %not_a0  = xor i64 %a0, -1
+  %not_a1  = xor i64 %a1, -1
+  %not_a2  = xor i64 %a2, -1
+  %or0     = or i64 %not_a0, %not_a1
+  %or1     = or i64 %a0, %not_a2
+  %and     = and i64 %or0, %or1
+  ret i64 %and
+}
+
+define i32 @not_a_masked_merge0_demorgan(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-LABEL: not_a_masked_merge0_demorgan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    orl %edi, %edx
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    negl %eax
+; CHECK-NEXT:    orl %esi, %eax
+; CHECK-NEXT:    andl %edx, %eax
+; CHECK-NEXT:    retq
+  %not_a_not = sub i32 0, %a0
+  %or0 = or i32 %not_a_not, %a1
+  %or1 = or i32 %a0, %a2
+  %and = and i32 %or0, %or1
+  ret i32 %and
+}
+
+; not a masked merge: `not` operand does not match another `and`-operand.
+define i32 @not_a_masked_merge1_demorgan(i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+; NOBMI-LABEL: not_a_masked_merge1_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %ecx, %eax
+; NOBMI-NEXT:    orl %edx, %edi
+; NOBMI-NEXT:    notl %eax
+; NOBMI-NEXT:    orl %esi, %eax
+; NOBMI-NEXT:    andl %edi, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: not_a_masked_merge1_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl %edx, %edi
+; BMI-NEXT:    andnl %ecx, %esi, %eax
+; BMI-NEXT:    andnl %edi, %eax, %eax
+; BMI-NEXT:    retq
+  %or1 = or i32 %a0, %a2
+  %not = xor i32 %a3, -1
+  %or0 = or i32 %not, %a1
+  %and = and i32 %or0, %or1
+  ret i32 %and
+}
+
+; not a masked merge: one of the operands of `and` is not an `or`.
+define i32 @not_a_masked_merge2_demorgan(i32 %a0, i32 %a1, i32 %a2) {
+; NOBMI-LABEL: not_a_masked_merge2_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %edi, %eax
+; NOBMI-NEXT:    andl %edi, %edx
+; NOBMI-NEXT:    notl %eax
+; NOBMI-NEXT:    orl %esi, %eax
+; NOBMI-NEXT:    andl %edx, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: not_a_masked_merge2_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    andl %edi, %edx
+; BMI-NEXT:    andnl %edi, %esi, %eax
+; BMI-NEXT:    andnl %edx, %eax, %eax
+; BMI-NEXT:    retq
+  %not_an_or1 = and i32 %a0, %a2
+  %not = xor i32 %a0, -1
+  %or0 = or i32 %not, %a1
+  %and = and i32 %or0, %not_an_or1
+  ret i32 %and
+}
+
+define i32 @not_a_masked_merge3_demorgan(i32 %a0, i32 %a1, i32 %a2) {
+; NOBMI-LABEL: not_a_masked_merge3_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %esi, %eax
+; NOBMI-NEXT:    orl %edi, %edx
+; NOBMI-NEXT:    xorl %edi, %eax
+; NOBMI-NEXT:    notl %eax
+; NOBMI-NEXT:    andl %edx, %eax
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: not_a_masked_merge3_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl %edi, %edx
+; BMI-NEXT:    xorl %edi, %esi
+; BMI-NEXT:    andnl %edx, %esi, %eax
+; BMI-NEXT:    retq
+  %or1 = or i32 %a0, %a2
+  %not = xor i32 %a0, -1
+  %not_an_or0 = xor i32 %not, %a1
+  %and = and i32 %not_an_or0, %or1
+  ret i32 %and
+}
+
+; not a masked merge: `not` operand must not be on same `or`.
+define i32 @not_a_masked_merge4_demorgan(i32 %a0, i32 %a1, i32 %a2) {
+; CHECK-LABEL: not_a_masked_merge4_demorgan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edi, %eax
+; CHECK-NEXT:    orl %edx, %eax
+; CHECK-NEXT:    retq
+  %or1 = or i32 %a0, %a2
+  %not = xor i32 %a1, -1
+  %or0 = or i32 %not, %a1
+  %and = and i32 %or0, %or1
+  ret i32 %and
+}
+
+; should not transform when operands have multiple users.
+define i32 @masked_merge_no_transform0_demorgan(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; NOBMI-LABEL: masked_merge_no_transform0_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    orl %edi, %edx
+; NOBMI-NEXT:    movl %edi, %eax
+; NOBMI-NEXT:    notl %eax
+; NOBMI-NEXT:    orl %esi, %eax
+; NOBMI-NEXT:    andl %edx, %eax
+; NOBMI-NEXT:    movl %edx, (%rcx)
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: masked_merge_no_transform0_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl %edi, %edx
+; BMI-NEXT:    andnl %edi, %esi, %eax
+; BMI-NEXT:    andnl %edx, %eax, %eax
+; BMI-NEXT:    movl %edx, (%rcx)
+; BMI-NEXT:    retq
+  %not = xor i32 %a0, -1
+  %or0 = or i32 %not, %a1
+  %or1 = or i32 %a0, %a2
+  %and = and i32 %or0, %or1
+  store i32 %or1, ptr %p1
+  ret i32 %and
+}
+
+; should not transform when operands have multiple users.
+define i32 @masked_merge_no_transform1_demorgan(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; NOBMI-LABEL: masked_merge_no_transform1_demorgan:
+; NOBMI:       # %bb.0:
+; NOBMI-NEXT:    movl %edx, %eax
+; NOBMI-NEXT:    orl %edi, %eax
+; NOBMI-NEXT:    notl %edi
+; NOBMI-NEXT:    orl %edi, %esi
+; NOBMI-NEXT:    andl %esi, %eax
+; NOBMI-NEXT:    movl %edi, (%rcx)
+; NOBMI-NEXT:    retq
+;
+; BMI-LABEL: masked_merge_no_transform1_demorgan:
+; BMI:       # %bb.0:
+; BMI-NEXT:    orl %edi, %edx
+; BMI-NEXT:    andnl %edi, %esi, %eax
+; BMI-NEXT:    notl %edi
+; BMI-NEXT:    andnl %edx, %eax, %eax
+; BMI-NEXT:    movl %edi, (%rcx)
+; BMI-NEXT:    retq
+  %not = xor i32 %a0, -1
+  %or0 = or i32 %not, %a1
+  %or1 = or i32 %a0, %a2
+  %and = and i32 %or0, %or1
+  store i32 %not, ptr %p1
+  ret i32 %and
+}
+
+; should not transform when operands have multiple users.
+define i32 @masked_merge_no_transform2_demorgan(i32 %a0, i32 %a1, i32 %a2, ptr %p1) {
+; CHECK-LABEL: masked_merge_no_transform2_demorgan:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl %edx, %eax
+; CHECK-NEXT:    orl %edi, %eax
+; CHECK-NEXT:    notl %edi
+; CHECK-NEXT:    orl %esi, %edi
+; CHECK-NEXT:    andl %edi, %eax
+; CHECK-NEXT:    movl %edi, (%rcx)
+; CHECK-NEXT:    retq
+  %not = xor i32 %a0, -1
+  %or0 = or i32 %not, %a1
+  %or1 = or i32 %a0, %a2
+  %and = and i32 %or0, %or1
+  store i32 %or0, ptr %p1
+  ret i32 %and
+}
-- 
cgit v1.2.3


From fe3933da15b5bc635bce156f1f8d11a784316a07 Mon Sep 17 00:00:00 2001
From: Yang Bai <baiyang0132@gmail.com>
Date: Thu, 19 Jun 2025 00:26:04 +0800
Subject: [mlir][vector] Support complete folding in single pass for
 vector.insert/vector.extract (#142124)

### Description

This patch improves the folding efficiency of `vector.insert` and
`vector.extract` operations by not returning early after successfully
converting dynamic indices to static indices.

This PR also renames the test pass `TestConstantFold` to
`TestSingleFold` and adds comprehensive documentation explaining the
single-pass folding behavior.

### Motivation

Since the `OpBuilder::createOrFold` function only calls `fold` **once**,
the current `fold` methods of `vector.insert` and `vector.extract` may
leave the op in a state that can be folded further. For example,
consider the following un-folded IR:
```
%v1 = vector.insert %e1, %v0 [0] : f32 into vector<128xf32>
%c0 = arith.constant 0 : index
%e2 = vector.extract %v1[%c0] : f32 from vector<128xf32>
```
If we use `createOrFold` to create the `vector.extract` op, then the
result will be:
```
%v1 = vector.insert %e1, %v0 [127] : f32 into vector<128xf32>
%e2 = vector.extract %v1[0] : f32 from vector<128xf32>
```
But this is not the optimal result. `createOrFold` should have returned
`%e1`.
The reason is that the execution of fold returns immediately after
`extractInsertFoldConstantOp`, causing subsequent folding logics to be
skipped.

---------

Co-authored-by: Yang Bai <yangb@nvidia.com>
---
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp          | 24 ++++---
 mlir/test/Dialect/Affine/constant-fold.mlir       |  2 +-
 mlir/test/Dialect/Linalg/mesh-spmdization.mlir    |  2 +-
 mlir/test/Dialect/Mesh/spmdization.mlir           |  2 +-
 mlir/test/Dialect/Tensor/mesh-spmdization.mlir    |  2 +-
 mlir/test/Dialect/Tosa/constant_folding.mlir      |  2 +-
 mlir/test/Dialect/Vector/constant-fold.mlir       |  4 +-
 mlir/test/Dialect/Vector/single-fold.mlir         | 38 ++++++++++
 mlir/test/Transforms/constant-fold-debuginfo.mlir |  2 +-
 mlir/test/Transforms/constant-fold.mlir           |  2 +-
 mlir/test/lib/Transforms/CMakeLists.txt           |  2 +-
 mlir/test/lib/Transforms/TestConstantFold.cpp     | 75 --------------------
 mlir/test/lib/Transforms/TestSingleFold.cpp       | 85 +++++++++++++++++++++++
 mlir/tools/mlir-opt/mlir-opt.cpp                  |  4 +-
 14 files changed, 150 insertions(+), 96 deletions(-)
 create mode 100644 mlir/test/Dialect/Vector/single-fold.mlir
 delete mode 100644 mlir/test/lib/Transforms/TestConstantFold.cpp
 create mode 100644 mlir/test/lib/Transforms/TestSingleFold.cpp

diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 2a2357319bd2..e576eeac2365 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -2063,6 +2063,7 @@ static Value extractInsertFoldConstantOp(OpType op, AdaptorType adaptor,
   if (opChange) {
     op.setStaticPosition(staticPosition);
     op.getOperation()->setOperands(operands);
+    // Return the original result to indicate an in-place folding happened.
     return op.getResult();
   }
   return {};
@@ -2146,11 +2147,12 @@ OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
     return getVector();
   if (auto res = foldPoisonSrcExtractOp(adaptor.getVector()))
     return res;
-  // Fold `arith.constant` indices into the `vector.extract` operation. Make
-  // sure that patterns requiring constant indices are added after this fold.
+  // Fold `arith.constant` indices into the `vector.extract` operation.
+  // Do not stop here as this fold may enable subsequent folds that require
+  // constant indices.
   SmallVector<Value> operands = {getVector()};
-  if (auto val = extractInsertFoldConstantOp(*this, adaptor, operands))
-    return val;
+  auto inplaceFolded = extractInsertFoldConstantOp(*this, adaptor, operands);
+
   if (auto res = foldPoisonIndexInsertExtractOp(
           getContext(), adaptor.getStaticPosition(), kPoisonIndex))
     return res;
@@ -2172,7 +2174,8 @@ OpFoldResult ExtractOp::fold(FoldAdaptor adaptor) {
     return val;
   if (auto val = foldScalarExtractFromFromElements(*this))
     return val;
-  return OpFoldResult();
+
+  return inplaceFolded;
 }
 
 namespace {
@@ -3272,11 +3275,12 @@ OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
   // (type mismatch).
   if (getNumIndices() == 0 && getValueToStoreType() == getType())
     return getValueToStore();
-  // Fold `arith.constant` indices into the `vector.insert` operation. Make
-  // sure that patterns requiring constant indices are added after this fold.
+  // Fold `arith.constant` indices into the `vector.insert` operation.
+  // Do not stop here as this fold may enable subsequent folds that require
+  // constant indices.
   SmallVector<Value> operands = {getValueToStore(), getDest()};
-  if (auto val = extractInsertFoldConstantOp(*this, adaptor, operands))
-    return val;
+  auto inplaceFolded = extractInsertFoldConstantOp(*this, adaptor, operands);
+
   if (auto res = foldPoisonIndexInsertExtractOp(
           getContext(), adaptor.getStaticPosition(), kPoisonIndex))
     return res;
@@ -3286,7 +3290,7 @@ OpFoldResult vector::InsertOp::fold(FoldAdaptor adaptor) {
     return res;
   }
 
-  return {};
+  return inplaceFolded;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Affine/constant-fold.mlir b/mlir/test/Dialect/Affine/constant-fold.mlir
index ffc3946db08d..8bddacc02475 100644
--- a/mlir/test/Dialect/Affine/constant-fold.mlir
+++ b/mlir/test/Dialect/Affine/constant-fold.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -test-constant-fold -split-input-file %s | FileCheck %s
+// RUN: mlir-opt -test-single-fold -split-input-file %s | FileCheck %s
 
 // CHECK-LABEL: func @affine_apply
 func.func @affine_apply(%variable : index) -> (index, index, index) {
diff --git a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
index 487cec00de16..9805ee4ea552 100644
--- a/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
+++ b/mlir/test/Dialect/Linalg/mesh-spmdization.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt \
-// RUN:  --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
+// RUN:  --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-single-fold))" \
 // RUN:  --split-input-file \
 // RUN:  %s | FileCheck %s
 
diff --git a/mlir/test/Dialect/Mesh/spmdization.mlir b/mlir/test/Dialect/Mesh/spmdization.mlir
index 5c9fd29444f0..af4ab58ea50a 100644
--- a/mlir/test/Dialect/Mesh/spmdization.mlir
+++ b/mlir/test/Dialect/Mesh/spmdization.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt \
-// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
+// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-single-fold))" \
 // RUN:   %s | FileCheck %s
 
 mesh.mesh @mesh_1d(shape = 2)
diff --git a/mlir/test/Dialect/Tensor/mesh-spmdization.mlir b/mlir/test/Dialect/Tensor/mesh-spmdization.mlir
index 3fb842474550..8598d81ff6cf 100644
--- a/mlir/test/Dialect/Tensor/mesh-spmdization.mlir
+++ b/mlir/test/Dialect/Tensor/mesh-spmdization.mlir
@@ -1,5 +1,5 @@
 // RUN: mlir-opt \
-// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-constant-fold))" \
+// RUN:   --pass-pipeline="builtin.module(func.func(mesh-spmdization,test-single-fold))" \
 // RUN:   %s | FileCheck %s
 
 mesh.mesh @mesh_1d_4(shape = 4)
diff --git a/mlir/test/Dialect/Tosa/constant_folding.mlir b/mlir/test/Dialect/Tosa/constant_folding.mlir
index 9b6ccdb54c10..d477a2479e91 100644
--- a/mlir/test/Dialect/Tosa/constant_folding.mlir
+++ b/mlir/test/Dialect/Tosa/constant_folding.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt --test-constant-fold %s | FileCheck %s
+// RUN: mlir-opt --test-single-fold %s | FileCheck %s
 
 // CHECK-LABEL: func @test_const
 func.func @test_const(%arg0 : index) -> tensor<4xi32> {
diff --git a/mlir/test/Dialect/Vector/constant-fold.mlir b/mlir/test/Dialect/Vector/constant-fold.mlir
index 66c91d6b2041..cbb159fd59ff 100644
--- a/mlir/test/Dialect/Vector/constant-fold.mlir
+++ b/mlir/test/Dialect/Vector/constant-fold.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -test-constant-fold | FileCheck %s
+// RUN: mlir-opt %s -split-input-file -test-single-fold | FileCheck %s
 
 // CHECK-LABEL: fold_extract_transpose_negative
 func.func @fold_extract_transpose_negative(%arg0: vector<4x4xf16>) -> vector<4x4xf16> {
@@ -11,3 +11,5 @@ func.func @fold_extract_transpose_negative(%arg0: vector<4x4xf16>) -> vector<4x4
   %2 = vector.extract %1[0] : vector<4x4xf16> from vector<1x4x4xf16>
   return %2 : vector<4x4xf16>
 }
+
+
diff --git a/mlir/test/Dialect/Vector/single-fold.mlir b/mlir/test/Dialect/Vector/single-fold.mlir
new file mode 100644
index 000000000000..baccdc3f51c0
--- /dev/null
+++ b/mlir/test/Dialect/Vector/single-fold.mlir
@@ -0,0 +1,38 @@
+// RUN: mlir-opt %s -split-input-file -test-single-fold | FileCheck %s
+
+// The tests in this file verify that fold() methods can handle complex
+// optimization scenarios without requiring multiple folding iterations.
+// This is important because:
+//
+// 1. OpBuilder::createOrFold() only calls fold() once, so operations must
+//    be fully optimized in that single call
+// 2. Multiple rounds of folding would incur higher performance costs,
+//    so it's more efficient to complete all optimizations in one pass
+//
+// These tests ensure that folding implementations are robust and complete,
+// avoiding situations where operations are left in intermediate states
+// that could be further optimized.
+
+// CHECK-LABEL: fold_extract_in_single_pass
+// CHECK-SAME: (%{{.*}}: vector<4xf16>, %[[ARG1:.+]]: f16)
+func.func @fold_extract_in_single_pass(%arg0: vector<4xf16>, %arg1: f16) -> f16 {
+  %0 = vector.insert %arg1, %arg0 [1] : f16 into vector<4xf16>
+  %c1 = arith.constant 1 : index
+  // Verify that the fold is finished in a single pass even if the index is dynamic.
+  %1 = vector.extract %0[%c1] : f16 from vector<4xf16>
+  // CHECK: return %[[ARG1]] : f16
+  return %1 : f16
+}
+
+// -----
+
+// CHECK-LABEL: fold_insert_in_single_pass
+func.func @fold_insert_in_single_pass() -> vector<2xf16> {
+  %cst = arith.constant dense<0.000000e+00> : vector<2xf16>
+  %c1 = arith.constant 1 : index
+  %c2 = arith.constant 2.5 : f16
+  // Verify that the fold is finished in a single pass even if the index is dynamic.
+  // CHECK: arith.constant dense<[0.000000e+00, 2.500000e+00]> : vector<2xf16>
+  %0 = vector.insert %c2, %cst [%c1] : f16 into vector<2xf16>
+  return %0 : vector<2xf16>
+} 
\ No newline at end of file
diff --git a/mlir/test/Transforms/constant-fold-debuginfo.mlir b/mlir/test/Transforms/constant-fold-debuginfo.mlir
index c308bc477bee..4fa7fb6698a2 100644
--- a/mlir/test/Transforms/constant-fold-debuginfo.mlir
+++ b/mlir/test/Transforms/constant-fold-debuginfo.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -split-input-file -test-constant-fold -mlir-print-debuginfo | FileCheck %s
+// RUN: mlir-opt %s -split-input-file -test-single-fold -mlir-print-debuginfo | FileCheck %s
 
 // CHECK-LABEL: func @fold_and_merge
 func.func @fold_and_merge() -> (i32, i32) {
diff --git a/mlir/test/Transforms/constant-fold.mlir b/mlir/test/Transforms/constant-fold.mlir
index 981757aed9b1..0b393bf0556b 100644
--- a/mlir/test/Transforms/constant-fold.mlir
+++ b/mlir/test/Transforms/constant-fold.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -test-constant-fold | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -test-single-fold | FileCheck %s
 
 // -----
 
diff --git a/mlir/test/lib/Transforms/CMakeLists.txt b/mlir/test/lib/Transforms/CMakeLists.txt
index 76041cd6cd79..ddc0a779e8f6 100644
--- a/mlir/test/lib/Transforms/CMakeLists.txt
+++ b/mlir/test/lib/Transforms/CMakeLists.txt
@@ -26,11 +26,11 @@ endif()
 add_mlir_library(MLIRTestTransforms
   TestCommutativityUtils.cpp
   TestCompositePass.cpp
-  TestConstantFold.cpp
   TestControlFlowSink.cpp
   TestInlining.cpp
   TestInliningCallback.cpp
   TestMakeIsolatedFromAbove.cpp
+  TestSingleFold.cpp
   TestTransformsOps.cpp
   ${MLIRTestTransformsPDLSrc}
 
diff --git a/mlir/test/lib/Transforms/TestConstantFold.cpp b/mlir/test/lib/Transforms/TestConstantFold.cpp
deleted file mode 100644
index c97ab9091cb6..000000000000
--- a/mlir/test/lib/Transforms/TestConstantFold.cpp
+++ /dev/null
@@ -1,75 +0,0 @@
-//===- TestConstantFold.cpp - Pass to test constant folding ---------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "mlir/Pass/Pass.h"
-#include "mlir/Transforms/FoldUtils.h"
-
-using namespace mlir;
-
-namespace {
-/// Simple constant folding pass.
-struct TestConstantFold : public PassWrapper<TestConstantFold, OperationPass<>>,
-                          public RewriterBase::Listener {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestConstantFold)
-
-  StringRef getArgument() const final { return "test-constant-fold"; }
-  StringRef getDescription() const final {
-    return "Test operation constant folding";
-  }
-  // All constants in the operation post folding.
-  SmallVector<Operation *> existingConstants;
-
-  void foldOperation(Operation *op, OperationFolder &helper);
-  void runOnOperation() override;
-
-  void notifyOperationInserted(Operation *op,
-                               OpBuilder::InsertPoint previous) override {
-    existingConstants.push_back(op);
-  }
-  void notifyOperationErased(Operation *op) override {
-    auto *it = llvm::find(existingConstants, op);
-    if (it != existingConstants.end())
-      existingConstants.erase(it);
-  }
-};
-} // namespace
-
-void TestConstantFold::foldOperation(Operation *op, OperationFolder &helper) {
-  // Attempt to fold the specified operation, including handling unused or
-  // duplicated constants.
-  (void)helper.tryToFold(op);
-}
-
-void TestConstantFold::runOnOperation() {
-  existingConstants.clear();
-
-  // Collect and fold the operations within the operation.
-  SmallVector<Operation *, 8> ops;
-  getOperation()->walk<mlir::WalkOrder::PreOrder>([&](Operation *op) { ops.push_back(op); });
-
-  // Fold the constants in reverse so that the last generated constants from
-  // folding are at the beginning. This creates somewhat of a linear ordering to
-  // the newly generated constants that matches the operation order and improves
-  // the readability of test cases.
-  OperationFolder helper(&getContext(), /*listener=*/this);
-  for (Operation *op : llvm::reverse(ops))
-    foldOperation(op, helper);
-
-  // By the time we are done, we may have simplified a bunch of code, leaving
-  // around dead constants. Check for them now and remove them.
-  for (auto *cst : existingConstants) {
-    if (cst->use_empty())
-      cst->erase();
-  }
-}
-
-namespace mlir {
-namespace test {
-void registerTestConstantFold() { PassRegistration<TestConstantFold>(); }
-} // namespace test
-} // namespace mlir
diff --git a/mlir/test/lib/Transforms/TestSingleFold.cpp b/mlir/test/lib/Transforms/TestSingleFold.cpp
new file mode 100644
index 000000000000..5bd9dd2a1f07
--- /dev/null
+++ b/mlir/test/lib/Transforms/TestSingleFold.cpp
@@ -0,0 +1,85 @@
+//===- TestSingleFold.cpp - Pass to test single-pass folding --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Pass/Pass.h"
+#include "mlir/Transforms/FoldUtils.h"
+
+using namespace mlir;
+
+namespace {
+/// Test pass for single-pass constant folding.
+///
+/// This pass tests the behavior of operations when folded exactly once. Unlike
+/// canonicalization passes that may apply multiple rounds of folding, this pass
+/// ensures that each operation is folded at most once, which is useful for
+/// testing scenarios where the fold implementation should handle complex cases
+/// without requiring multiple iterations.
+///
+/// The pass also removes dead constants after folding to clean up unused
+/// intermediate results.
+struct TestSingleFold : public PassWrapper<TestSingleFold, OperationPass<>>,
+                        public RewriterBase::Listener {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSingleFold)
+
+  StringRef getArgument() const final { return "test-single-fold"; }
+  StringRef getDescription() const final {
+    return "Test single-pass operation folding and dead constant elimination";
+  }
+  // All constants in the operation post folding.
+  SmallVector<Operation *> existingConstants;
+
+  void foldOperation(Operation *op, OperationFolder &helper);
+  void runOnOperation() override;
+
+  void notifyOperationInserted(Operation *op,
+                               OpBuilder::InsertPoint previous) override {
+    existingConstants.push_back(op);
+  }
+  void notifyOperationErased(Operation *op) override {
+    auto *it = llvm::find(existingConstants, op);
+    if (it != existingConstants.end())
+      existingConstants.erase(it);
+  }
+};
+} // namespace
+
+void TestSingleFold::foldOperation(Operation *op, OperationFolder &helper) {
+  // Attempt to fold the specified operation, including handling unused or
+  // duplicated constants.
+  (void)helper.tryToFold(op);
+}
+
+void TestSingleFold::runOnOperation() {
+  existingConstants.clear();
+
+  // Collect and fold the operations within the operation.
+  SmallVector<Operation *, 8> ops;
+  getOperation()->walk<mlir::WalkOrder::PreOrder>(
+      [&](Operation *op) { ops.push_back(op); });
+
+  // Fold the constants in reverse so that the last generated constants from
+  // folding are at the beginning. This creates somewhat of a linear ordering to
+  // the newly generated constants that matches the operation order and improves
+  // the readability of test cases.
+  OperationFolder helper(&getContext(), /*listener=*/this);
+  for (Operation *op : llvm::reverse(ops))
+    foldOperation(op, helper);
+
+  // By the time we are done, we may have simplified a bunch of code, leaving
+  // around dead constants. Check for them now and remove them.
+  for (auto *cst : existingConstants) {
+    if (cst->use_empty())
+      cst->erase();
+  }
+}
+
+namespace mlir {
+namespace test {
+void registerTestSingleFold() { PassRegistration<TestSingleFold>(); }
+} // namespace test
+} // namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index 6ef9ff8e8454..143a5e8e8f8d 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -87,7 +87,6 @@ void registerTestCfAssertPass();
 void registerTestCFGLoopInfoPass();
 void registerTestComposeSubView();
 void registerTestCompositePass();
-void registerTestConstantFold();
 void registerTestControlFlowSink();
 void registerTestConvertToSPIRVPass();
 void registerTestDataLayoutPropagation();
@@ -145,6 +144,7 @@ void registerTestSCFUtilsPass();
 void registerTestSCFWhileOpBuilderPass();
 void registerTestSCFWrapInZeroTripCheckPasses();
 void registerTestShapeMappingPass();
+void registerTestSingleFold();
 void registerTestSliceAnalysisPass();
 void registerTestSPIRVCPURunnerPipeline();
 void registerTestSPIRVFuncSignatureConversion();
@@ -233,7 +233,6 @@ void registerTestPasses() {
   mlir::test::registerTestCFGLoopInfoPass();
   mlir::test::registerTestComposeSubView();
   mlir::test::registerTestCompositePass();
-  mlir::test::registerTestConstantFold();
   mlir::test::registerTestControlFlowSink();
   mlir::test::registerTestConvertToSPIRVPass();
   mlir::test::registerTestDataLayoutPropagation();
@@ -291,6 +290,7 @@ void registerTestPasses() {
   mlir::test::registerTestSCFWhileOpBuilderPass();
   mlir::test::registerTestSCFWrapInZeroTripCheckPasses();
   mlir::test::registerTestShapeMappingPass();
+  mlir::test::registerTestSingleFold();
   mlir::test::registerTestSliceAnalysisPass();
   mlir::test::registerTestSPIRVCPURunnerPipeline();
   mlir::test::registerTestSPIRVFuncSignatureConversion();
-- 
cgit v1.2.3


From 4084ffcf1e69b962e864aa138bb54dabbcec912f Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Wed, 18 Jun 2025 11:31:03 -0500
Subject: [flang] Show types in DumpEvExpr (#143743)

When dumping evaluate::Expr, show type names which contain a lot of
useful information.

For example show
```
expr <Fortran::evaluate::SomeType> {
  expr <Fortran::evaluate::SomeKind<Fortran::common::TypeCategory::Integer>> {
    expr <Fortran::evaluate::Type<Fortran::common::TypeCategory::Integer, 4>> {
      ...
```
instead of
```
expr T {
  expr T {
    expr T {
      ...
```
---
 flang/include/flang/Semantics/dump-expr.h | 56 ++++++++++++++++++++++++++-----
 flang/lib/Semantics/dump-expr.cpp         |  2 +-
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/flang/include/flang/Semantics/dump-expr.h b/flang/include/flang/Semantics/dump-expr.h
index 2f445429a10b..9cc52b4da487 100644
--- a/flang/include/flang/Semantics/dump-expr.h
+++ b/flang/include/flang/Semantics/dump-expr.h
@@ -16,6 +16,7 @@
 
 #include <memory>
 #include <optional>
+#include <string>
 #include <variant>
 #include <vector>
 
@@ -38,6 +39,43 @@ public:
   }
 
 private:
+  template <typename T> struct TypeOf {
+    static constexpr std::string_view get() {
+#if defined(__GNUC__)
+#define DUMP_EXPR_SHOW_TYPE
+      std::string_view v(__PRETTY_FUNCTION__);
+      // Extract the "xyz" from the "pretty function" string:
+      // "... [with T = xyz; std::string_view = ...]"
+      std::string_view front("with T = ");
+      std::string_view back("; std::string_view =");
+
+#elif defined(_MSC_VER)
+#define DUMP_EXPR_SHOW_TYPE
+      std::string_view v(__FUNCSIG__);
+      // Extract the "xyz" from the "pretty function" string:
+      // "...TypeOf<xyz>::get(void)"
+      std::string_view front("TypeOf<");
+      std::string_view back(">::get(void)");
+
+#endif
+
+#if defined(DUMP_EXPR_SHOW_TYPE)
+#undef DUMP_EXPR_SHOW_TYPE
+      if (auto fpos{v.find(front)}; fpos != v.npos) {
+        v.remove_prefix(fpos + front.size());
+        if (auto bpos{v.find(back)}; bpos != v.npos) {
+          v.remove_suffix(v.size() - bpos);
+          return v;
+        }
+      }
+#endif
+
+      return "";
+    }
+
+    static constexpr std::string_view name{TypeOf<T>::get()};
+  };
+
   template <typename A, bool C> void Show(const common::Indirection<A, C> &x) {
     Show(x.value());
   }
@@ -76,7 +114,7 @@ private:
   void Show(const evaluate::NullPointer &);
   template <typename T> void Show(const evaluate::Constant<T> &x) {
     if constexpr (T::category == common::TypeCategory::Derived) {
-      Indent("derived constant");
+      Indent("derived constant "s + std::string(TypeOf<T>::name));
       for (const auto &map : x.values()) {
         for (const auto &pair : map) {
           Show(pair.second.value());
@@ -84,7 +122,7 @@ private:
       }
       Outdent();
     } else {
-      Print("constant");
+      Print("constant "s + std::string(TypeOf<T>::name));
     }
   }
   void Show(const Symbol &symbol);
@@ -102,7 +140,7 @@ private:
   void Show(const evaluate::Substring &x);
   void Show(const evaluate::ComplexPart &x);
   template <typename T> void Show(const evaluate::Designator<T> &x) {
-    Indent("designator");
+    Indent("designator "s + std::string(TypeOf<T>::name));
     Show(x.u);
     Outdent();
   }
@@ -117,7 +155,7 @@ private:
     Outdent();
   }
   template <typename T> void Show(const evaluate::FunctionRef<T> &x) {
-    Indent("function ref");
+    Indent("function ref "s + std::string(TypeOf<T>::name));
     Show(x.proc());
     Show(x.arguments());
     Outdent();
@@ -127,14 +165,14 @@ private:
   }
   template <typename T>
   void Show(const evaluate::ArrayConstructorValues<T> &x) {
-    Indent("array constructor value");
+    Indent("array constructor value "s + std::string(TypeOf<T>::name));
     for (auto &v : x) {
       Show(v);
     }
     Outdent();
   }
   template <typename T> void Show(const evaluate::ImpliedDo<T> &x) {
-    Indent("implied do");
+    Indent("implied do "s + std::string(TypeOf<T>::name));
     Show(x.lower());
     Show(x.upper());
     Show(x.stride());
@@ -148,20 +186,20 @@ private:
   void Show(const evaluate::StructureConstructor &x);
   template <typename D, typename R, typename O>
   void Show(const evaluate::Operation<D, R, O> &op) {
-    Indent("unary op");
+    Indent("unary op "s + std::string(TypeOf<D>::name));
     Show(op.left());
     Outdent();
   }
   template <typename D, typename R, typename LO, typename RO>
   void Show(const evaluate::Operation<D, R, LO, RO> &op) {
-    Indent("binary op");
+    Indent("binary op "s + std::string(TypeOf<D>::name));
     Show(op.left());
     Show(op.right());
     Outdent();
   }
   void Show(const evaluate::Relational<evaluate::SomeType> &x);
   template <typename T> void Show(const evaluate::Expr<T> &x) {
-    Indent("expr T");
+    Indent("expr <" + std::string(TypeOf<T>::name) + ">");
     Show(x.u);
     Outdent();
   }
diff --git a/flang/lib/Semantics/dump-expr.cpp b/flang/lib/Semantics/dump-expr.cpp
index aa0b4e0f0339..66cedab94bfb 100644
--- a/flang/lib/Semantics/dump-expr.cpp
+++ b/flang/lib/Semantics/dump-expr.cpp
@@ -151,7 +151,7 @@ void DumpEvaluateExpr::Show(const evaluate::StructureConstructor &x) {
 }
 
 void DumpEvaluateExpr::Show(const evaluate::Relational<evaluate::SomeType> &x) {
-  Indent("expr some type");
+  Indent("relational some type");
   Show(x.u);
   Outdent();
 }
-- 
cgit v1.2.3


From 2a8c65e983b3f4e1c83d8028d354f7bacc149015 Mon Sep 17 00:00:00 2001
From: Alexis Engelke <engelke@in.tum.de>
Date: Wed, 18 Jun 2025 18:56:30 +0200
Subject: [CodeGen][NFC] Fix quadratic c-t for large jump tables
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Deleting a basic block removes all references from jump tables, which
is O(n). When freeing a MachineFunction, all basic blocks are deleted
before the jump tables, causing O(n^2) runtime. Fix this by deallocating
the jump table first.

Test case generator:

    import sys

    n = int(sys.argv[1])
    print("define void @f(i64 %c, ptr %p) {")
    print("  switch i64 %c, label %d [")
    for i in range(n):
        print(f"    i64 {i}, label %h{i}")
    print(f"  ]")
    for i in range(n):
        print(f'h{i}:')
        print(f'  store i64 {i*i}, ptr %p')
        print(f'  ret void')
    print('d:')
    print('  ret void')
    print('}')

Improvement at 5000 entries:

    Benchmark 1: ./llc.pre -filetype=obj -O0 <switch5k.bc
      Time (mean ± σ):      49.7 ms ±   1.0 ms
      Range (min … max):    48.0 ms …  52.1 ms    57 runs

    Benchmark 2: ./llc.post -filetype=obj -O0 <switch5k.bc
      Time (mean ± σ):      39.4 ms ±   0.8 ms
      Range (min … max):    37.1 ms …  41.1 ms    72 runs

    Summary
      ./llc.post -filetype=obj -O0 <switch5k.bc ran
        1.26 ± 0.04 times faster than ./llc.pre -filetype=obj -O0 <switch5k.bc

Improvement at 20000 entries:

    Benchmark 1: ./llc.pre -filetype=obj -O0 <switch20k.bc
      Time (mean ± σ):     281.7 ms ±   1.0 ms
      Range (min … max):   280.2 ms … 283.0 ms    10 runs

    Benchmark 2: ./llc.post -filetype=obj -O0 <switch20k.bc
      Time (mean ± σ):     123.9 ms ±   1.5 ms
      Range (min … max):   121.4 ms … 129.2 ms    23 runs

    Summary
      ./llc.post -filetype=obj -O0 <switch20k.bc ran
        2.27 ± 0.03 times faster than ./llc.pre -filetype=obj -O0 <switch20k.bc

Pull Request: https://github.com/llvm/llvm-project/pull/144108
---
 llvm/lib/CodeGen/MachineFunction.cpp | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp
index 607e87a38274..38ad582ba923 100644
--- a/llvm/lib/CodeGen/MachineFunction.cpp
+++ b/llvm/lib/CodeGen/MachineFunction.cpp
@@ -259,6 +259,15 @@ MachineFunction::~MachineFunction() {
 
 void MachineFunction::clear() {
   Properties.reset();
+
+  // Clear JumpTableInfo first. Otherwise, every MBB we delete would do a
+  // linear search over the jump table entries to find and erase itself.
+  if (JumpTableInfo) {
+    JumpTableInfo->~MachineJumpTableInfo();
+    Allocator.Deallocate(JumpTableInfo);
+    JumpTableInfo = nullptr;
+  }
+
   // Don't call destructors on MachineInstr and MachineOperand. All of their
   // memory comes from the BumpPtrAllocator which is about to be purged.
   //
@@ -287,11 +296,6 @@ void MachineFunction::clear() {
   ConstantPool->~MachineConstantPool();
   Allocator.Deallocate(ConstantPool);
 
-  if (JumpTableInfo) {
-    JumpTableInfo->~MachineJumpTableInfo();
-    Allocator.Deallocate(JumpTableInfo);
-  }
-
   if (WinEHInfo) {
     WinEHInfo->~WinEHFuncInfo();
     Allocator.Deallocate(WinEHInfo);
-- 
cgit v1.2.3


From 77bc25485135b8a8cb2427910a8850fbc4e4be09 Mon Sep 17 00:00:00 2001
From: John Brawn <john.brawn@arm.com>
Date: Wed, 18 Jun 2025 18:05:02 +0100
Subject: [AArch64] Fix build failure with -Werror (#144749)

PR#144387 caused buildbot failures with -Werror due to a comparison
between signed and unsigned types. Fix this with an explicit cast.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 0e28ccd0f655..d8b574719dae 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27081,7 +27081,7 @@ bool AArch64TargetLowering::getIndexedAddressParts(SDNode *N, SDNode *Op,
     // only allow an offset that's equal to the store size.
     EVT MemType = cast<MemSDNode>(N)->getMemoryVT();
     if (!Subtarget->isLittleEndian() && MemType.isVector() &&
-        RHSC != MemType.getStoreSize())
+        (uint64_t)RHSC != MemType.getStoreSize())
       return false;
     // Always emit pre-inc/post-inc addressing mode. Use negated constant offset
     // when dealing with subtraction.
-- 
cgit v1.2.3


From 298f1c276f4f9c18b25a79ffe6e619e89c5fbf7e Mon Sep 17 00:00:00 2001
From: Artem Belevich <tra@google.com>
Date: Wed, 18 Jun 2025 10:08:27 -0700
Subject: Revert "Add missing intrinsics to cuda headers" (#144755)

Reverts llvm/llvm-project#143664
as it breaks CUDA compilation.
---
 clang/lib/Headers/__clang_cuda_intrinsics.h | 284 ----------------------------
 1 file changed, 284 deletions(-)

diff --git a/clang/lib/Headers/__clang_cuda_intrinsics.h b/clang/lib/Headers/__clang_cuda_intrinsics.h
index 5e13f3f78df7..8b230af6f664 100644
--- a/clang/lib/Headers/__clang_cuda_intrinsics.h
+++ b/clang/lib/Headers/__clang_cuda_intrinsics.h
@@ -479,290 +479,6 @@ inline __device__ unsigned __funnelshift_rc(unsigned low32, unsigned high32,
   return ret;
 }
 
-#pragma push_macro("__INTRINSIC_LOAD")
-#define __INTRINSIC_LOAD(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType,  \
-                         __Clobber)                                            \
-  inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
-    __TmpType __ret;                                                           \
-    asm(__AsmOp " %0, [%1];" : __AsmType(__ret) : "l"(__ptr)__Clobber);        \
-    return (__DeclType)__ret;                                                  \
-  }
-
-#pragma push_macro("__INTRINSIC_LOAD2")
-#define __INTRINSIC_LOAD2(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType, \
-                          __Clobber)                                           \
-  inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
-    __DeclType __ret;                                                          \
-    __TmpType __tmp;                                                           \
-    asm(__AsmOp " {%0,%1}, [%2];"                                              \
-        : __AsmType(__tmp.x), __AsmType(__tmp.y)                               \
-        : "l"(__ptr)__Clobber);                                                \
-    using __ElementType = decltype(__ret.x);                                   \
-    __ret.x = (__ElementType)(__tmp.x);                                        \
-    __ret.y = (__ElementType)__tmp.y;                                          \
-    return __ret;                                                              \
-  }
-
-#pragma push_macro("__INTRINSIC_LOAD4")
-#define __INTRINSIC_LOAD4(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType, \
-                          __Clobber)                                           \
-  inline __device__ __DeclType __FnName(const __DeclType *__ptr) {             \
-    __DeclType __ret;                                                          \
-    __TmpType __tmp;                                                           \
-    asm(__AsmOp " {%0,%1,%2,%3}, [%4];"                                        \
-        : __AsmType(__tmp.x), __AsmType(__tmp.y), __AsmType(__tmp.z),          \
-          __AsmType(__tmp.w)                                                   \
-        : "l"(__ptr)__Clobber);                                                \
-    using __ElementType = decltype(__ret.x);                                   \
-    __ret.x = (__ElementType)__tmp.x;                                          \
-    __ret.y = (__ElementType)__tmp.y;                                          \
-    __ret.z = (__ElementType)__tmp.z;                                          \
-    __ret.w = (__ElementType)__tmp.w;                                          \
-    return __ret;                                                              \
-  }
-
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s8", char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s8", signed char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s16", short, unsigned short, "=h", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s32", int, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.s64", long long, unsigned long long,
-                 "=l", );
-
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s8", char2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s8", char4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s16", short2, short2, "=h", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s16", short4, short4, "=h", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s32", int2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.s32", int4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.s64 ", longlong2, longlong2, "=l", );
-
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u8", unsigned char, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u16", unsigned short, unsigned short,
-                 "=h", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u32", unsigned int, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.u64", unsigned long long,
-                 unsigned long long, "=l", );
-
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u8", uchar2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u8", uchar4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u16", ushort2, ushort2, "=h", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u16", ushort4, ushort4, "=h", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u32", uint2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.u32", uint4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.u64", ulonglong2, ulonglong2,
-                  "=l", );
-
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.f32", float, float, "=f", );
-__INTRINSIC_LOAD(__ldcg, "ld.global.cg.f64", double, double, "=d", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.f32", float2, float2, "=f", );
-__INTRINSIC_LOAD4(__ldcg, "ld.global.cg.v4.f32", float4, float4, "=f", );
-__INTRINSIC_LOAD2(__ldcg, "ld.global.cg.v2.f64", double2, double2, "=d", );
-
-inline __device__ long __ldcg(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cg.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cg.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
-  }
-  return (long)__ret;
-}
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u8", unsigned char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u16", unsigned short, unsigned short,
-                 "=h", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u32", unsigned int, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.u64", unsigned long long,
-                 unsigned long long, "=l", : "memory");
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s8", char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s8", signed char, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s16", short, unsigned short,
-                 "=h", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s32", int, unsigned int,
-                 "=r", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.s64", long long, unsigned long long,
-                 "=l", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u8", uchar2, uint2,
-                  "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u8", uchar4, uint4,
-                  "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u16", ushort2, ushort2,
-                  "=h", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u16", ushort4, ushort4,
-                  "=h", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u32", uint2, uint2,
-                  "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.u32", uint4, uint4,
-                  "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.u64", ulonglong2, ulonglong2,
-                  "=l", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s8", char2, int2, "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s8", char4, int4, "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s16", short2, short2,
-                  "=h", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s16", short4, short4,
-                  "=h", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s32", int2, int2, "=r", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.s32", int4, int4, "=r", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.s64", longlong2, longlong2,
-                  "=l", : "memory");
-
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.f32", float, float, "=f", : "memory");
-__INTRINSIC_LOAD(__ldcv, "ld.global.cv.f64", double, double, "=d", : "memory");
-
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.f32", float2, float2,
-                  "=f", : "memory");
-__INTRINSIC_LOAD4(__ldcv, "ld.global.cv.v4.f32", float4, float4,
-                  "=f", : "memory");
-__INTRINSIC_LOAD2(__ldcv, "ld.global.cv.v2.f64", double2, double2,
-                  "=d", : "memory");
-
-inline __device__ long __ldcv(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cv.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cv.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
-  }
-  return (long)__ret;
-}
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s8", char, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s8", signed char, signed int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s16", short, unsigned short, "=h", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s32", int, unsigned int, "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.s64", long long, unsigned long long,
-                 "=l", );
-
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s8", char2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s8", char4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s16", short2, short2, "=h", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s16", short4, short4, "=h", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s32", int2, int2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.s32", int4, int4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.s64", longlong2, longlong2, "=l", );
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u8", unsigned char, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u16", unsigned short, unsigned short,
-                 "=h", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u32", unsigned int, unsigned int,
-                 "=r", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.u64", unsigned long long,
-                 unsigned long long, "=l", );
-
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u8", uchar2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u8", uchar4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u16", ushort2, ushort2, "=h", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u16", ushort4, ushort4, "=h", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u32", uint2, uint2, "=r", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.u32", uint4, uint4, "=r", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.u64", ulonglong2, ulonglong2,
-                  "=l", );
-
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.f32", float, float, "=f", );
-__INTRINSIC_LOAD(__ldcs, "ld.global.cs.f64", double, double, "=d", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.f32", float2, float2, "=f", );
-__INTRINSIC_LOAD4(__ldcs, "ld.global.cs.v4.f32", float4, float4, "=f", );
-__INTRINSIC_LOAD2(__ldcs, "ld.global.cs.v2.f64", double2, double2, "=d", );
-
-#pragma pop_macro("__INTRINSIC_LOAD")
-#pragma pop_macro("__INTRINSIC_LOAD2")
-#pragma pop_macro("__INTRINSIC_LOAD4")
-
-inline __device__ long __ldcs(const long *__ptr) {
-  unsigned long __ret;
-  if (sizeof(long) == 8) {
-    asm("ld.global.cs.s64 %0, [%1];" : "=l"(__ret) : "l"(__ptr));
-  } else {
-    asm("ld.global.cs.s32 %0, [%1];" : "=r"(__ret) : "l"(__ptr));
-  }
-  return (long)__ret;
-}
-
-#pragma push_macro("__INTRINSIC_STORE")
-#define __INTRINSIC_STORE(__FnName, __AsmOp, __DeclType, __TmpType, __AsmType) \
-  inline __device__ void __FnName(__DeclType *__ptr, __DeclType __value) {     \
-    __TmpType __tmp = (__TmpType)__value;                                      \
-    asm(__AsmOp " [%0], %1;" ::"l"(__ptr), __AsmType(__tmp) : "memory");       \
-  }
-
-#pragma push_macro("__INTRINSIC_STORE2")
-#define __INTRINSIC_STORE2(__FnName, __AsmOp, __DeclType, __TmpType,           \
-                           __AsmType)                                          \
-  inline __device__ void __FnName(__DeclType *__ptr, __DeclType __value) {     \
-    __TmpType __tmp;                                                           \
-    using __ElementType = decltype(__tmp.x);                                   \
-    __tmp.x = (__ElementType)(__value.x);                                      \
-    __tmp.y = (__ElementType)(__value.y);                                      \
-    asm(__AsmOp " [%0], {%1,%2};" ::"l"(__ptr), __AsmType(__tmp.x),            \
-        __AsmType(__tmp.y)                                                     \
-        : "memory");                                                           \
-  }
-
-#pragma push_macro("__INTRINSIC_STORE4")
-#define __INTRINSIC_STORE4(__FnName, __AsmOp, __DeclType, __TmpType,           \
-                           __AsmType)                                          \
-  inline __device__ void __FnName(__DeclType *__ptr, __DeclType __value) {     \
-    __TmpType __tmp;                                                           \
-    using __ElementType = decltype(__tmp.x);                                   \
-    __tmp.x = (__ElementType)(__value.x);                                      \
-    __tmp.y = (__ElementType)(__value.y);                                      \
-    __tmp.z = (__ElementType)(__value.z);                                      \
-    __tmp.w = (__ElementType)(__value.w);                                      \
-    asm(__AsmOp " [%0], {%1,%2,%3,%4};" ::"l"(__ptr), __AsmType(__tmp.x),      \
-        __AsmType(__tmp.y), __AsmType(__tmp.z), __AsmType(__tmp.w)             \
-        : "memory");                                                           \
-  }
-
-__INTRINSIC_STORE(__stwt, "st.global.wt.s8", char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s8", signed char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s16", short, short, "h");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s32", int, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.s64", long long, long long, "l");
-
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s8", char2, int2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s8", char4, int4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s16", short2, short2, "h");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s16", short4, short4, "h");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s32", int2, int2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.s32", int4, int4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.s64", longlong2, longlong2, "l");
-
-__INTRINSIC_STORE(__stwt, "st.global.wt.u8", unsigned char, int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u16", unsigned short, unsigned short,
-                  "h");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u32", unsigned int, unsigned int, "r");
-__INTRINSIC_STORE(__stwt, "st.global.wt.u64", unsigned long long,
-                  unsigned long long, "l");
-
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u8", uchar2, uchar2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u8", uchar4, uint4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u16", ushort2, ushort2, "h");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u16", ushort4, ushort4, "h");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u32", uint2, uint2, "r");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.u32", uint4, uint4, "r");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.u64", ulonglong2, ulonglong2, "l");
-
-__INTRINSIC_STORE(__stwt, "st.global.wt.f32", float, float, "f");
-__INTRINSIC_STORE(__stwt, "st.global.wt.f64", double, double, "d");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.f32", float2, float2, "f");
-__INTRINSIC_STORE4(__stwt, "st.global.wt.v4.f32", float4, float4, "f");
-__INTRINSIC_STORE2(__stwt, "st.global.wt.v2.f64", double2, double2, "d");
-
-#pragma pop_macro("__INTRINSIC_STORE")
-#pragma pop_macro("__INTRINSIC_STORE2")
-#pragma pop_macro("__INTRINSIC_STORE4")
-
 #endif // !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 320
 
 #if CUDA_VERSION >= 11000
-- 
cgit v1.2.3


From d9f7979a63ceac88727632ecfd522c073288b6c1 Mon Sep 17 00:00:00 2001
From: Justin King <jcking@google.com>
Date: Wed, 18 Jun 2025 10:24:38 -0700
Subject: sanitizer_common: add unsupported test for free_sized and
 free_aligned_sized from C23 (#144727)

Signed-off-by: Justin King <jcking@google.com>
---
 .../sanitizer_common/TestCases/Linux/free_aligned_sized.c | 13 +++++++++++++
 .../test/sanitizer_common/TestCases/Linux/free_sized.c    | 15 +++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
 create mode 100644 compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c

diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
new file mode 100644
index 000000000000..f4c6c0f973bd
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
@@ -0,0 +1,13 @@
+// RUN: %clang -std=c23 -O0 %s -o %t && %run %t
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, lsan, ubsan
+
+#include <stddef.h>
+#include <stdlib.h>
+
+extern void free_aligned_sized(void *p, size_t alignment, size_t size);
+
+int main() {
+  volatile void *p = aligned_alloc(128, 1024);
+  free_aligned_sized((void *)p, 128, 1024);
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
new file mode 100644
index 000000000000..0ee2289684d0
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
@@ -0,0 +1,15 @@
+// RUN: %clang -std=c23 -O0 %s -o %t && %run %t
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, lsan, ubsan
+
+#include <stddef.h>
+#include <stdlib.h>
+
+extern void *aligned_alloc(size_t alignment, size_t size);
+
+extern void free_sized(void *p, size_t size);
+
+int main() {
+  volatile void *p = malloc(64);
+  free_sized((void *)p, 64);
+  return 0;
+}
-- 
cgit v1.2.3


From 82acd8c377e9ed267195afdbde16eedebabc648c Mon Sep 17 00:00:00 2001
From: Lei Huang <lei@ca.ibm.com>
Date: Wed, 18 Jun 2025 13:50:57 -0400
Subject: [PowerPC]  Add code to spill and restore DMRp registers (#142443)

---
 llvm/lib/Target/PowerPC/PPCInstrInfo.cpp    |   2 +-
 llvm/lib/Target/PowerPC/PPCInstrInfo.h      |   9 ++
 llvm/lib/Target/PowerPC/PPCInstrMMA.td      |   4 +
 llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp | 107 +++++++-------
 llvm/test/CodeGen/PowerPC/dmr-spill.ll      |  36 ++---
 llvm/test/CodeGen/PowerPC/dmrp-spill.ll     | 213 ++++++++++++++++++++++++++++
 6 files changed, 300 insertions(+), 71 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/dmrp-spill.ll

diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 57c86d9e5de6..7c1550e99bae 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -1926,7 +1926,7 @@ unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
   } else if (PPC::DMRROWpRCRegClass.hasSubClassEq(RC)) {
     llvm_unreachable("TODO: Implement spill DMRROWp regclass!");
   } else if (PPC::DMRpRCRegClass.hasSubClassEq(RC)) {
-    llvm_unreachable("TODO: Implement spill DMRp regclass!");
+    OpcodeIndex = SOK_DMRpSpill;
   } else if (PPC::DMRRCRegClass.hasSubClassEq(RC)) {
     OpcodeIndex = SOK_DMRSpill;
   } else {
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index a27b5718ec89..7931a9e3ae13 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -81,6 +81,7 @@ enum SpillOpcodeKey {
   SOK_AccumulatorSpill,
   SOK_UAccumulatorSpill,
   SOK_WAccumulatorSpill,
+  SOK_DMRpSpill,
   SOK_DMRSpill,
   SOK_SPESpill,
   SOK_PairedG8Spill,
@@ -119,6 +120,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::EVLDD,                                                                 \
    PPC::RESTORE_QUADWORD}
 
@@ -140,6 +142,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
 
 #define Pwr10LoadOpcodes                                                       \
@@ -160,6 +163,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
 
 #define FutureLoadOpcodes                                                      \
@@ -178,6 +182,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::RESTORE_ACC,                                                           \
    PPC::RESTORE_UACC,                                                          \
    PPC::RESTORE_WACC,                                                          \
+   PPC::RESTORE_DMRP,                                                          \
    PPC::RESTORE_DMR,                                                           \
    NoInstr,                                                                    \
    PPC::RESTORE_QUADWORD}
@@ -199,6 +204,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::EVSTDD,                                                                \
    PPC::SPILL_QUADWORD}
 
@@ -220,6 +226,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
 
 #define Pwr10StoreOpcodes                                                      \
@@ -240,6 +247,7 @@ enum PPCMachineCombinerPattern : unsigned {
    NoInstr,                                                                    \
    NoInstr,                                                                    \
    NoInstr,                                                                    \
+   NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
 
 #define FutureStoreOpcodes                                                     \
@@ -258,6 +266,7 @@ enum PPCMachineCombinerPattern : unsigned {
    PPC::SPILL_ACC,                                                             \
    PPC::SPILL_UACC,                                                            \
    PPC::SPILL_WACC,                                                            \
+   PPC::SPILL_DMRP,                                                            \
    PPC::SPILL_DMR,                                                             \
    NoInstr,                                                                    \
    PPC::SPILL_QUADWORD}
diff --git a/llvm/lib/Target/PowerPC/PPCInstrMMA.td b/llvm/lib/Target/PowerPC/PPCInstrMMA.td
index 82e4a60e0a72..436715a0e4ab 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrMMA.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrMMA.td
@@ -565,12 +565,16 @@ let Predicates = [MMA, IsISAFuture], isCodeGenOnly = 1 in {
   let mayStore = 1 in {
     def SPILL_WACC: PPCEmitTimePseudo<(outs), (ins wacc:$AT, memrix16:$dst),
                                       "#SPILL_WACC", []>;
+    def SPILL_DMRP: PPCEmitTimePseudo<(outs), (ins dmrp:$AT, memrix16:$dst),
+                                      "#SPILL_DMRP", []>;
     def SPILL_DMR: PPCEmitTimePseudo<(outs), (ins dmr:$AT, memrix16:$dst),
                                       "#SPILL_DMR", []>;
   }
   let mayLoad = 1, hasSideEffects = 0 in {
     def RESTORE_WACC: PPCEmitTimePseudo<(outs wacc:$AT), (ins memrix16:$src),
                                         "#RESTORE_WACC", []>;
+    def RESTORE_DMRP: PPCEmitTimePseudo<(outs dmrp:$AT), (ins memrix16:$src),
+                                        "#RESTORE_DMRP", []>;
     def RESTORE_DMR: PPCEmitTimePseudo<(outs dmr:$AT), (ins memrix16:$src),
                                         "#RESTORE_DMR", []>;
   }
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
index ea34c1aba82e..76dca4794e05 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -1519,33 +1519,32 @@ void PPCRegisterInfo::lowerDMRSpilling(MachineBasicBlock::iterator II,
   // DMR is made up of WACC and WACC_HI, so DMXXEXTFDMR512 to spill
   // the corresponding 512 bits.
   const TargetRegisterClass *RC = &PPC::VSRpRCRegClass;
-  Register SrcReg = MI.getOperand(0).getReg();
-
-  Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg2 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg3 = MF.getRegInfo().createVirtualRegister(RC);
+  auto spillDMR = [&](Register SrcReg, int BEIdx, int LEIdx) {
+    auto spillWACC = [&](unsigned Opc, unsigned RegIdx, int IdxBE, int IdxLE) {
+      Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
+      Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
+
+      BuildMI(MBB, II, DL, TII.get(Opc), VSRpReg0)
+          .addDef(VSRpReg1)
+          .addReg(TargetRegisterInfo::getSubReg(SrcReg, RegIdx));
+
+      addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                            .addReg(VSRpReg0, RegState::Kill),
+                        FrameIndex, IsLittleEndian ? IdxLE : IdxBE);
+      addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
+                            .addReg(VSRpReg1, RegState::Kill),
+                        FrameIndex, IsLittleEndian ? IdxLE - 32 : IdxBE + 32);
+    };
+    spillWACC(PPC::DMXXEXTFDMR512, PPC::sub_wacc_lo, BEIdx, LEIdx);
+    spillWACC(PPC::DMXXEXTFDMR512_HI, PPC::sub_wacc_hi, BEIdx + 64, LEIdx - 64);
+  };
 
-  BuildMI(MBB, II, DL, TII.get(PPC::DMXXEXTFDMR512_HI), VSRpReg2)
-      .addDef(VSRpReg3)
-      .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_wacc_hi));
-
-  BuildMI(MBB, II, DL, TII.get(PPC::DMXXEXTFDMR512), VSRpReg0)
-      .addDef(VSRpReg1)
-      .addReg(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_wacc_lo));
-
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                        .addReg(VSRpReg0, RegState::Kill),
-                    FrameIndex, IsLittleEndian ? 96 : 0);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                        .addReg(VSRpReg1, RegState::Kill),
-                    FrameIndex, IsLittleEndian ? 64 : 32);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                        .addReg(VSRpReg2, RegState::Kill),
-                    FrameIndex, IsLittleEndian ? 32 : 64);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::STXVP))
-                        .addReg(VSRpReg3, RegState::Kill),
-                    FrameIndex, IsLittleEndian ? 0 : 96);
+  Register SrcReg = MI.getOperand(0).getReg();
+  if (MI.getOpcode() == PPC::SPILL_DMRP) {
+    spillDMR(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_dmr1), 0, 96);
+    spillDMR(TargetRegisterInfo::getSubReg(SrcReg, PPC::sub_dmr0), 128, 224);
+  } else
+    spillDMR(SrcReg, 0, 96);
 
   // Discard the pseudo instruction.
   MBB.erase(II);
@@ -1554,7 +1553,7 @@ void PPCRegisterInfo::lowerDMRSpilling(MachineBasicBlock::iterator II,
 /// lowerDMRRestore - Generate the code to restore the DMR register.
 void PPCRegisterInfo::lowerDMRRestore(MachineBasicBlock::iterator II,
                                       unsigned FrameIndex) const {
-  MachineInstr &MI = *II; // <DestReg> = RESTORE_WACC <offset>
+  MachineInstr &MI = *II; // <DestReg> = RESTORE_DMR[P] <offset>
   MachineBasicBlock &MBB = *MI.getParent();
   MachineFunction &MF = *MBB.getParent();
   const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
@@ -1563,32 +1562,34 @@ void PPCRegisterInfo::lowerDMRRestore(MachineBasicBlock::iterator II,
   bool IsLittleEndian = Subtarget.isLittleEndian();
 
   const TargetRegisterClass *RC = &PPC::VSRpRCRegClass;
-  Register DestReg = MI.getOperand(0).getReg();
-
-  Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg2 = MF.getRegInfo().createVirtualRegister(RC);
-  Register VSRpReg3 = MF.getRegInfo().createVirtualRegister(RC);
+  auto restoreDMR = [&](Register DestReg, int BEIdx, int LEIdx) {
+    auto restoreWACC = [&](unsigned Opc, unsigned RegIdx, int IdxBE,
+                           int IdxLE) {
+      Register VSRpReg0 = MF.getRegInfo().createVirtualRegister(RC);
+      Register VSRpReg1 = MF.getRegInfo().createVirtualRegister(RC);
+
+      addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg0),
+                        FrameIndex, IsLittleEndian ? IdxLE : IdxBE);
+      addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg1),
+                        FrameIndex, IsLittleEndian ? IdxLE - 32 : IdxBE + 32);
+
+      // Kill virtual registers (killedRegState::Killed).
+      BuildMI(MBB, II, DL, TII.get(Opc),
+              TargetRegisterInfo::getSubReg(DestReg, RegIdx))
+          .addReg(VSRpReg0, RegState::Kill)
+          .addReg(VSRpReg1, RegState::Kill);
+    };
+    restoreWACC(PPC::DMXXINSTDMR512, PPC::sub_wacc_lo, BEIdx, LEIdx);
+    restoreWACC(PPC::DMXXINSTDMR512_HI, PPC::sub_wacc_hi, BEIdx + 64,
+                LEIdx - 64);
+  };
 
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg0),
-                    FrameIndex, IsLittleEndian ? 96 : 0);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg1),
-                    FrameIndex, IsLittleEndian ? 64 : 32);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg2),
-                    FrameIndex, IsLittleEndian ? 32 : 64);
-  addFrameReference(BuildMI(MBB, II, DL, TII.get(PPC::LXVP), VSRpReg3),
-                    FrameIndex, IsLittleEndian ? 0 : 96);
-
-  // Kill virtual registers (killedRegState::Killed).
-  BuildMI(MBB, II, DL, TII.get(PPC::DMXXINSTDMR512_HI),
-          TargetRegisterInfo::getSubReg(DestReg, PPC::sub_wacc_hi))
-      .addReg(VSRpReg2, RegState::Kill)
-      .addReg(VSRpReg3, RegState::Kill);
-
-  BuildMI(MBB, II, DL, TII.get(PPC::DMXXINSTDMR512),
-          TargetRegisterInfo::getSubReg(DestReg, PPC::sub_wacc_lo))
-      .addReg(VSRpReg0, RegState::Kill)
-      .addReg(VSRpReg1, RegState::Kill);
+  Register DestReg = MI.getOperand(0).getReg();
+  if (MI.getOpcode() == PPC::RESTORE_DMRP) {
+    restoreDMR(TargetRegisterInfo::getSubReg(DestReg, PPC::sub_dmr1), 0, 96);
+    restoreDMR(TargetRegisterInfo::getSubReg(DestReg, PPC::sub_dmr0), 128, 224);
+  } else
+    restoreDMR(DestReg, 0, 96);
 
   // Discard the pseudo instruction.
   MBB.erase(II);
@@ -1756,9 +1757,11 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   case PPC::RESTORE_WACC:
     lowerWACCRestore(II, FrameIndex);
     return true;
+  case PPC::SPILL_DMRP:
   case PPC::SPILL_DMR:
     lowerDMRSpilling(II, FrameIndex);
     return true;
+  case PPC::RESTORE_DMRP:
   case PPC::RESTORE_DMR:
     lowerDMRRestore(II, FrameIndex);
     return true;
diff --git a/llvm/test/CodeGen/PowerPC/dmr-spill.ll b/llvm/test/CodeGen/PowerPC/dmr-spill.ll
index c1b01cd2d3fd..983fce4127af 100644
--- a/llvm/test/CodeGen/PowerPC/dmr-spill.ll
+++ b/llvm/test/CodeGen/PowerPC/dmr-spill.ll
@@ -30,19 +30,19 @@ define void @spillDMRreg(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) nounwind {
 ; CHECK-NEXT:    lxv v3, 0(r4)
 ; CHECK-NEXT:    lxv vs0, 0(r5)
 ; CHECK-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; CHECK-NEXT:    stxvp vsp36, 128(r1)
+; CHECK-NEXT:    stxvp vsp34, 96(r1)
 ; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
-; CHECK-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
-; CHECK-NEXT:    stxvp vsp38, 128(r1)
-; CHECK-NEXT:    stxvp vsp32, 96(r1)
 ; CHECK-NEXT:    stxvp vsp36, 64(r1)
 ; CHECK-NEXT:    stxvp vsp34, 32(r1)
 ; CHECK-NEXT:    bl dummy_func@notoc
 ; CHECK-NEXT:    lxvp vsp34, 128(r1)
 ; CHECK-NEXT:    lxvp vsp36, 96(r1)
-; CHECK-NEXT:    lxvp vsp32, 64(r1)
-; CHECK-NEXT:    lxvp vsp38, 32(r1)
-; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
 ; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; CHECK-NEXT:    lxvp vsp34, 64(r1)
+; CHECK-NEXT:    lxvp vsp36, 32(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
 ; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
 ; CHECK-NEXT:    stxvp vsp34, 96(r30)
 ; CHECK-NEXT:    stxvp vsp36, 64(r30)
@@ -72,20 +72,20 @@ define void @spillDMRreg(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) nounwind {
 ; AIX-NEXT:    lxv v3, 16(r4)
 ; AIX-NEXT:    lxv vs0, 0(r5)
 ; AIX-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; AIX-NEXT:    stxvp vsp36, 112(r1)
+; AIX-NEXT:    stxvp vsp34, 144(r1)
 ; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
-; AIX-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
-; AIX-NEXT:    stxvp vsp38, 112(r1)
-; AIX-NEXT:    stxvp vsp32, 144(r1)
 ; AIX-NEXT:    stxvp vsp36, 176(r1)
 ; AIX-NEXT:    stxvp vsp34, 208(r1)
 ; AIX-NEXT:    bl .dummy_func[PR]
 ; AIX-NEXT:    nop
 ; AIX-NEXT:    lxvp vsp34, 112(r1)
 ; AIX-NEXT:    lxvp vsp36, 144(r1)
-; AIX-NEXT:    lxvp vsp32, 176(r1)
-; AIX-NEXT:    lxvp vsp38, 208(r1)
-; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
 ; AIX-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX-NEXT:    lxvp vsp34, 176(r1)
+; AIX-NEXT:    lxvp vsp36, 208(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
 ; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
 ; AIX-NEXT:    stxvp vsp36, 96(r31)
 ; AIX-NEXT:    stxvp vsp34, 64(r31)
@@ -115,20 +115,20 @@ define void @spillDMRreg(ptr %vop, ptr %vpp, ptr %vcp, ptr %resp) nounwind {
 ; AIX32-NEXT:    lxv v3, 16(r4)
 ; AIX32-NEXT:    lxv vs0, 0(r5)
 ; AIX32-NEXT:    dmxvbf16gerx2pp dmr0, vsp34, vs0
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; AIX32-NEXT:    stxvp vsp36, 64(r1)
+; AIX32-NEXT:    stxvp vsp34, 96(r1)
 ; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
-; AIX32-NEXT:    dmxxextfdmr512 vsp38, vsp32, wacc0, 0
-; AIX32-NEXT:    stxvp vsp38, 64(r1)
-; AIX32-NEXT:    stxvp vsp32, 96(r1)
 ; AIX32-NEXT:    stxvp vsp36, 128(r1)
 ; AIX32-NEXT:    stxvp vsp34, 160(r1)
 ; AIX32-NEXT:    bl .dummy_func[PR]
 ; AIX32-NEXT:    nop
 ; AIX32-NEXT:    lxvp vsp34, 64(r1)
 ; AIX32-NEXT:    lxvp vsp36, 96(r1)
-; AIX32-NEXT:    lxvp vsp32, 128(r1)
-; AIX32-NEXT:    lxvp vsp38, 160(r1)
-; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp32, vsp38, 1
 ; AIX32-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX32-NEXT:    lxvp vsp34, 128(r1)
+; AIX32-NEXT:    lxvp vsp36, 160(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
 ; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
 ; AIX32-NEXT:    stxvp vsp36, 96(r31)
 ; AIX32-NEXT:    stxvp vsp34, 64(r31)
diff --git a/llvm/test/CodeGen/PowerPC/dmrp-spill.ll b/llvm/test/CodeGen/PowerPC/dmrp-spill.ll
new file mode 100644
index 000000000000..62d42d4a26d5
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/dmrp-spill.ll
@@ -0,0 +1,213 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
+; RUN:   -disable-auto-paired-vec-st=false -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-aix \
+; RUN:   -disable-auto-paired-vec-st=false -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s --check-prefix=AIX
+; RUN: llc -verify-machineinstrs -mtriple=powerpc-unknown-aix \
+; RUN:   -disable-auto-paired-vec-st=false -ppc-asm-full-reg-names \
+; RUN:   -ppc-vsr-nums-as-vr -mcpu=future < %s | FileCheck %s --check-prefix=AIX32
+
+declare void @dummy_func()
+declare <2048 x i1> @llvm.ppc.mma.dmsha3hash(<2048 x i1>, i32)
+
+define dso_local void @test_dmsha3hash(ptr %vopp, ptr %resp) nounwind {
+; CHECK-LABEL: test_dmsha3hash:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    mflr r0
+; CHECK-NEXT:    std r0, 16(r1)
+; CHECK-NEXT:    stdu r1, -304(r1)
+; CHECK-NEXT:    std r30, 288(r1) # 8-byte Folded Spill
+; CHECK-NEXT:    lxvp vsp34, 0(r3)
+; CHECK-NEXT:    lxvp vsp36, 32(r3)
+; CHECK-NEXT:    mr r30, r4
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi1, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 64(r3)
+; CHECK-NEXT:    lxvp vsp36, 96(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc1, vsp36, vsp34, 0
+; CHECK-NEXT:    lxvp vsp34, 128(r3)
+; CHECK-NEXT:    lxvp vsp36, 160(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; CHECK-NEXT:    lxvp vsp34, 192(r3)
+; CHECK-NEXT:    lxvp vsp36, 224(r3)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; CHECK-NEXT:    dmsha3hash dmrp0, 5
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc1, 0
+; CHECK-NEXT:    stxvp vsp36, 128(r1)
+; CHECK-NEXT:    stxvp vsp34, 96(r1)
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi1, 1
+; CHECK-NEXT:    stxvp vsp36, 64(r1)
+; CHECK-NEXT:    stxvp vsp34, 32(r1)
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; CHECK-NEXT:    stxvp vsp36, 256(r1)
+; CHECK-NEXT:    stxvp vsp34, 224(r1)
+; CHECK-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp36, 192(r1)
+; CHECK-NEXT:    stxvp vsp34, 160(r1)
+; CHECK-NEXT:    bl dummy_func@notoc
+; CHECK-NEXT:    lxvp vsp34, 128(r1)
+; CHECK-NEXT:    lxvp vsp36, 96(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc1, vsp34, vsp36, 0
+; CHECK-NEXT:    lxvp vsp34, 64(r1)
+; CHECK-NEXT:    lxvp vsp36, 32(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi1, vsp34, vsp36, 1
+; CHECK-NEXT:    lxvp vsp34, 256(r1)
+; CHECK-NEXT:    lxvp vsp36, 224(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; CHECK-NEXT:    lxvp vsp34, 192(r1)
+; CHECK-NEXT:    lxvp vsp36, 160(r1)
+; CHECK-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; CHECK-NEXT:    stxvp vsp34, 224(r30)
+; CHECK-NEXT:    stxvp vsp36, 192(r30)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; CHECK-NEXT:    stxvp vsp34, 160(r30)
+; CHECK-NEXT:    stxvp vsp36, 128(r30)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc1, 0
+; CHECK-NEXT:    stxvp vsp34, 96(r30)
+; CHECK-NEXT:    stxvp vsp36, 64(r30)
+; CHECK-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi1, 1
+; CHECK-NEXT:    stxvp vsp34, 32(r30)
+; CHECK-NEXT:    stxvp vsp36, 0(r30)
+; CHECK-NEXT:    ld r30, 288(r1) # 8-byte Folded Reload
+; CHECK-NEXT:    addi r1, r1, 304
+; CHECK-NEXT:    ld r0, 16(r1)
+; CHECK-NEXT:    mtlr r0
+; CHECK-NEXT:    blr
+;
+; AIX-LABEL: test_dmsha3hash:
+; AIX:       # %bb.0: # %entry
+; AIX-NEXT:    mflr r0
+; AIX-NEXT:    std r0, 16(r1)
+; AIX-NEXT:    stdu r1, -384(r1)
+; AIX-NEXT:    std r31, 376(r1) # 8-byte Folded Spill
+; AIX-NEXT:    lxvp vsp34, 224(r3)
+; AIX-NEXT:    lxvp vsp36, 192(r3)
+; AIX-NEXT:    mr r31, r4
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi1, vsp36, vsp34, 1
+; AIX-NEXT:    lxvp vsp34, 160(r3)
+; AIX-NEXT:    lxvp vsp36, 128(r3)
+; AIX-NEXT:    dmxxinstdmr512 wacc1, vsp36, vsp34, 0
+; AIX-NEXT:    lxvp vsp34, 96(r3)
+; AIX-NEXT:    lxvp vsp36, 64(r3)
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; AIX-NEXT:    lxvp vsp34, 32(r3)
+; AIX-NEXT:    lxvp vsp36, 0(r3)
+; AIX-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; AIX-NEXT:    dmsha3hash dmrp0, 5
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc1, 0
+; AIX-NEXT:    stxvp vsp36, 112(r1)
+; AIX-NEXT:    stxvp vsp34, 144(r1)
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi1, 1
+; AIX-NEXT:    stxvp vsp36, 176(r1)
+; AIX-NEXT:    stxvp vsp34, 208(r1)
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; AIX-NEXT:    stxvp vsp36, 240(r1)
+; AIX-NEXT:    stxvp vsp34, 272(r1)
+; AIX-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; AIX-NEXT:    stxvp vsp36, 304(r1)
+; AIX-NEXT:    stxvp vsp34, 336(r1)
+; AIX-NEXT:    bl .dummy_func[PR]
+; AIX-NEXT:    nop
+; AIX-NEXT:    lxvp vsp34, 112(r1)
+; AIX-NEXT:    lxvp vsp36, 144(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc1, vsp34, vsp36, 0
+; AIX-NEXT:    lxvp vsp34, 176(r1)
+; AIX-NEXT:    lxvp vsp36, 208(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi1, vsp34, vsp36, 1
+; AIX-NEXT:    lxvp vsp34, 240(r1)
+; AIX-NEXT:    lxvp vsp36, 272(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX-NEXT:    lxvp vsp34, 304(r1)
+; AIX-NEXT:    lxvp vsp36, 336(r1)
+; AIX-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi1, 1
+; AIX-NEXT:    stxvp vsp36, 224(r31)
+; AIX-NEXT:    stxvp vsp34, 192(r31)
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc1, 0
+; AIX-NEXT:    stxvp vsp36, 160(r31)
+; AIX-NEXT:    stxvp vsp34, 128(r31)
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; AIX-NEXT:    stxvp vsp36, 96(r31)
+; AIX-NEXT:    stxvp vsp34, 64(r31)
+; AIX-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; AIX-NEXT:    stxvp vsp36, 32(r31)
+; AIX-NEXT:    stxvp vsp34, 0(r31)
+; AIX-NEXT:    ld r31, 376(r1) # 8-byte Folded Reload
+; AIX-NEXT:    addi r1, r1, 384
+; AIX-NEXT:    ld r0, 16(r1)
+; AIX-NEXT:    mtlr r0
+; AIX-NEXT:    blr
+;
+; AIX32-LABEL: test_dmsha3hash:
+; AIX32:       # %bb.0: # %entry
+; AIX32-NEXT:    mflr r0
+; AIX32-NEXT:    stw r0, 8(r1)
+; AIX32-NEXT:    stwu r1, -336(r1)
+; AIX32-NEXT:    stw r31, 332(r1) # 4-byte Folded Spill
+; AIX32-NEXT:    lxvp vsp34, 224(r3)
+; AIX32-NEXT:    lxvp vsp36, 192(r3)
+; AIX32-NEXT:    mr r31, r4
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi1, vsp36, vsp34, 1
+; AIX32-NEXT:    lxvp vsp34, 160(r3)
+; AIX32-NEXT:    lxvp vsp36, 128(r3)
+; AIX32-NEXT:    dmxxinstdmr512 wacc1, vsp36, vsp34, 0
+; AIX32-NEXT:    lxvp vsp34, 96(r3)
+; AIX32-NEXT:    lxvp vsp36, 64(r3)
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp36, vsp34, 1
+; AIX32-NEXT:    lxvp vsp34, 32(r3)
+; AIX32-NEXT:    lxvp vsp36, 0(r3)
+; AIX32-NEXT:    dmxxinstdmr512 wacc0, vsp36, vsp34, 0
+; AIX32-NEXT:    dmsha3hash dmrp0, 5
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc1, 0
+; AIX32-NEXT:    stxvp vsp36, 64(r1)
+; AIX32-NEXT:    stxvp vsp34, 96(r1)
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi1, 1
+; AIX32-NEXT:    stxvp vsp36, 128(r1)
+; AIX32-NEXT:    stxvp vsp34, 160(r1)
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc0, 0
+; AIX32-NEXT:    stxvp vsp36, 192(r1)
+; AIX32-NEXT:    stxvp vsp34, 224(r1)
+; AIX32-NEXT:    dmxxextfdmr512 vsp36, vsp34, wacc_hi0, 1
+; AIX32-NEXT:    stxvp vsp36, 256(r1)
+; AIX32-NEXT:    stxvp vsp34, 288(r1)
+; AIX32-NEXT:    bl .dummy_func[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lxvp vsp34, 64(r1)
+; AIX32-NEXT:    lxvp vsp36, 96(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc1, vsp34, vsp36, 0
+; AIX32-NEXT:    lxvp vsp34, 128(r1)
+; AIX32-NEXT:    lxvp vsp36, 160(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi1, vsp34, vsp36, 1
+; AIX32-NEXT:    lxvp vsp34, 192(r1)
+; AIX32-NEXT:    lxvp vsp36, 224(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc0, vsp34, vsp36, 0
+; AIX32-NEXT:    lxvp vsp34, 256(r1)
+; AIX32-NEXT:    lxvp vsp36, 288(r1)
+; AIX32-NEXT:    dmxxinstdmr512 wacc_hi0, vsp34, vsp36, 1
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi1, 1
+; AIX32-NEXT:    stxvp vsp36, 224(r31)
+; AIX32-NEXT:    stxvp vsp34, 192(r31)
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc1, 0
+; AIX32-NEXT:    stxvp vsp36, 160(r31)
+; AIX32-NEXT:    stxvp vsp34, 128(r31)
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc_hi0, 1
+; AIX32-NEXT:    stxvp vsp36, 96(r31)
+; AIX32-NEXT:    stxvp vsp34, 64(r31)
+; AIX32-NEXT:    dmxxextfdmr512 vsp34, vsp36, wacc0, 0
+; AIX32-NEXT:    stxvp vsp36, 32(r31)
+; AIX32-NEXT:    stxvp vsp34, 0(r31)
+; AIX32-NEXT:    lwz r31, 332(r1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi r1, r1, 336
+; AIX32-NEXT:    lwz r0, 8(r1)
+; AIX32-NEXT:    mtlr r0
+; AIX32-NEXT:    blr
+  entry:
+    %0 = load <2048 x i1>, ptr %vopp, align 64
+    %2 = tail call <2048 x i1> @llvm.ppc.mma.dmsha3hash(<2048 x i1> %0, i32 5)
+    tail call void @dummy_func()
+    %3 = tail call <2048 x i1> @llvm.ppc.mma.dmsha3hash(<2048 x i1> %0, i32 5)
+    store <2048 x i1> %2, ptr %resp, align 64
+    ret void
+}
-- 
cgit v1.2.3


From 835d3034fe96931cf907537b51b9cdd87b59d3ad Mon Sep 17 00:00:00 2001
From: Tomer Shafir <tomer.shafir8@gmail.com>
Date: Wed, 18 Jun 2025 20:56:33 +0300
Subject: [AArch64] improve zero-cycle regmov test (#143680)

- Add a `gpr32` suffix to test name to denote the specific register
class being checked
- Expand `-mtriple=arm64-apple-ios` to `-march=arm64` to broaden the
test context to the generic architecture, as the specific triple is not
required
- Port `bl` match to Linux too via the regex: `{{_?foo}}`
- Advance `-mcpu=cyclone` to the newer M series major `-mcpu=apple-m1`
- Use `-mcpu` so that `-mattr=-zcm` has a real effect
- Add a test that generic arm64 doesn't optimize for ZCM
- Distinguish 4 different assembly layouts: NOTCPU, CPU, NOTATTR, ATTR
- Fix broken test logic, for example: `; NOT: mov [[REG2:w[0-9]+]], w3`
matched `mov w1, w3` then `REG2` captured `w1` but then `; NOT: mov w1,
[[REG2]]` matched by prefix `mov, w1, w19` even though it should have
matched `mov w1, w1`. This change adds explicit matches for all of the
generated copies.
---
 .../AArch64/arm64-zero-cycle-regmov-gpr32.ll       | 45 ++++++++++++++++++++++
 .../CodeGen/AArch64/arm64-zero-cycle-regmov.ll     | 23 -----------
 2 files changed, 45 insertions(+), 23 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll

diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
new file mode 100644
index 000000000000..5ef6d3e84805
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov-gpr32.ll
@@ -0,0 +1,45 @@
+; RUN: llc < %s -march=arm64 | FileCheck %s -check-prefixes=NOTCPU --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 | FileCheck %s -check-prefixes=CPU --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mcpu=apple-m1 -mattr=-zcm | FileCheck %s -check-prefixes=NOTATTR --match-full-lines
+; RUN: llc < %s -mtriple=arm64-apple-macosx -mattr=+zcm | FileCheck %s -check-prefixes=ATTR --match-full-lines
+
+define void @t(i32 %a, i32 %b, i32 %c, i32 %d) {
+entry:
+; CHECK-LABEL: t:
+; NOTCPU: mov w0, w2
+; NOTCPU: mov w1, w3
+; NOTCPU: mov [[REG2:w[0-9]+]], w3
+; NOTCPU: mov [[REG1:w[0-9]+]], w2
+; NOTCPU-NEXT: bl {{_?foo}}
+; NOTCPU: mov w0, [[REG1]]
+; NOTCPU: mov w1, [[REG2]]
+
+; CPU: mov [[REG2:x[0-9]+]], x3
+; CPU: mov [[REG1:x[0-9]+]], x2
+; CPU: mov x0, x2
+; CPU: mov x1, x3
+; CPU-NEXT: bl {{_?foo}}
+; CPU: mov x0, [[REG1]]
+; CPU: mov x1, [[REG2]]
+
+; NOTATTR: mov [[REG2:w[0-9]+]], w3
+; NOTATTR: mov [[REG1:w[0-9]+]], w2
+; NOTATTR: mov w0, w2
+; NOTATTR: mov w1, w3
+; NOTATTR-NEXT: bl {{_?foo}}
+; NOTATTR: mov w0, [[REG1]]
+; NOTATTR: mov w1, [[REG2]]
+
+; ATTR: mov x0, x2
+; ATTR: mov x1, x3
+; ATTR: mov [[REG2:x[0-9]+]], x3
+; ATTR: mov [[REG1:x[0-9]+]], x2
+; ATTR-NEXT: bl {{_?foo}}
+; ATTR: mov x0, [[REG1]]
+; ATTR: mov x1, [[REG2]]
+  %call = call i32 @foo(i32 %c, i32 %d)
+  %call1 = call i32 @foo(i32 %c, i32 %d)
+  unreachable
+}
+
+declare i32 @foo(i32, i32)
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll
deleted file mode 100644
index b390853d44bf..000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll
+++ /dev/null
@@ -1,23 +0,0 @@
-; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=-zcm   | FileCheck %s -check-prefixes=CHECK,NOT
-; RUN: llc < %s -mtriple=arm64-apple-ios -mattr=+zcm   | FileCheck %s -check-prefixes=CHECK,YES
-; RUN: llc < %s -mtriple=arm64-apple-ios -mcpu=cyclone | FileCheck %s -check-prefixes=CHECK,YES
-
-; rdar://12254953
-define i32 @t(i32 %a, i32 %b, i32 %c, i32 %d) nounwind ssp {
-entry:
-; CHECK-LABEL: t:
-; NOT: mov [[REG2:w[0-9]+]], w3
-; NOT: mov [[REG1:w[0-9]+]], w2
-; YES: mov [[REG2:x[0-9]+]], x3
-; YES: mov [[REG1:x[0-9]+]], x2
-; CHECK: bl _foo
-; NOT: mov w0, [[REG1]]
-; NOT: mov w1, [[REG2]]
-; YES: mov x0, [[REG1]]
-; YES: mov x1, [[REG2]]
-  %call = call i32 @foo(i32 %c, i32 %d) nounwind
-  %call1 = call i32 @foo(i32 %c, i32 %d) nounwind
-  unreachable
-}
-
-declare i32 @foo(i32, i32)
-- 
cgit v1.2.3


From 6f4e4ea17745d1414519651eb4067ce14031ea93 Mon Sep 17 00:00:00 2001
From: sribee8 <sriya.pratipati@gmail.com>
Date: Wed, 18 Jun 2025 10:56:57 -0700
Subject: [libc] Internal getrandom implementation (#144427)

Implemented an internal getrandom to avoid calls to the public one in
table.h

---------

Co-authored-by: Sriya Pratipati <sriyap@google.com>
---
 libc/src/__support/HashTable/CMakeLists.txt    |  3 ++-
 libc/src/__support/HashTable/randomness.h      | 16 ++++++------
 libc/src/__support/OSUtil/linux/CMakeLists.txt | 13 ++++++++++
 libc/src/__support/OSUtil/linux/getrandom.h    | 35 ++++++++++++++++++++++++++
 libc/src/sys/random/linux/getrandom.cpp        | 13 +++++-----
 5 files changed, 63 insertions(+), 17 deletions(-)
 create mode 100644 libc/src/__support/OSUtil/linux/getrandom.h

diff --git a/libc/src/__support/HashTable/CMakeLists.txt b/libc/src/__support/HashTable/CMakeLists.txt
index a1de0680cc7d..698b8d0dfa68 100644
--- a/libc/src/__support/HashTable/CMakeLists.txt
+++ b/libc/src/__support/HashTable/CMakeLists.txt
@@ -15,7 +15,8 @@ if (NOT ${getrandom_index} EQUAL -1)
   message(STATUS "Using getrandom for hashtable randomness")
   set(randomness_compile_flags -DLIBC_HASHTABLE_USE_GETRANDOM)
   set(randomness_extra_depends
-    libc.src.sys.random.getrandom libc.src.errno.errno)
+    libc.src.__support.OSUtil.linux.getrandom
+    libc.hdr.errno_macros)
 endif()
 
 
diff --git a/libc/src/__support/HashTable/randomness.h b/libc/src/__support/HashTable/randomness.h
index 6b58a4125f78..7e54c9aa6ad1 100644
--- a/libc/src/__support/HashTable/randomness.h
+++ b/libc/src/__support/HashTable/randomness.h
@@ -14,8 +14,8 @@
 #include "src/__support/macros/attributes.h"
 #include "src/__support/macros/config.h"
 #if defined(LIBC_HASHTABLE_USE_GETRANDOM)
-#include "src/__support/libc_errno.h"
-#include "src/sys/random/getrandom.h"
+#include "hdr/errno_macros.h"
+#include "src/__support/OSUtil/linux/getrandom.h"
 #endif
 
 namespace LIBC_NAMESPACE_DECL {
@@ -35,20 +35,18 @@ LIBC_INLINE uint64_t next_random_seed() {
     entropy[0] = reinterpret_cast<uint64_t>(&entropy);
     entropy[1] = reinterpret_cast<uint64_t>(&state);
 #if defined(LIBC_HASHTABLE_USE_GETRANDOM)
-    int errno_backup = libc_errno;
     size_t count = sizeof(entropy);
     uint8_t *buffer = reinterpret_cast<uint8_t *>(entropy);
     while (count > 0) {
-      ssize_t len = getrandom(buffer, count, 0);
-      if (len == -1) {
-        if (libc_errno == ENOSYS)
+      auto len = internal::getrandom(buffer, count, 0);
+      if (!len.has_value()) {
+        if (len.error() == ENOSYS)
           break;
         continue;
       }
-      count -= len;
-      buffer += len;
+      count -= len.value();
+      buffer += len.value();
     }
-    libc_errno = errno_backup;
 #endif
     state.update(&entropy, sizeof(entropy));
   }
diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt
index 4681d8c2bb73..f303e54ce7b3 100644
--- a/libc/src/__support/OSUtil/linux/CMakeLists.txt
+++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt
@@ -24,6 +24,19 @@ add_object_library(
     libc.include.sys_syscall
 )
 
+add_header_library(
+  getrandom
+  HDRS
+    getrandom.h
+  DEPENDS
+    libc.src.__support.OSUtil.osutil
+    libc.src.__support.common
+    libc.src.__support.error_or
+    libc.src.__support.macros.config
+    libc.hdr.types.ssize_t
+    libc.include.sys_syscall
+)
+
 add_header_library(
   vdso_sym
   HDRS
diff --git a/libc/src/__support/OSUtil/linux/getrandom.h b/libc/src/__support/OSUtil/linux/getrandom.h
new file mode 100644
index 000000000000..793639472fee
--- /dev/null
+++ b/libc/src/__support/OSUtil/linux/getrandom.h
@@ -0,0 +1,35 @@
+//===------------ Implementation of getrandom function ----------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_GETRANDOM_H
+#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_GETRANDOM_H
+
+#include "hdr/types/ssize_t.h"
+#include "src/__support/OSUtil/linux/syscall.h" // syscall_impl
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include <sys/syscall.h> // For syscall numbers
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+LIBC_INLINE static ErrorOr<ssize_t> getrandom(void *buf, size_t buflen,
+                                              unsigned int flags) {
+  ssize_t ret =
+      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_getrandom, buf, buflen, flags);
+  if (ret < 0) {
+    return Error(-static_cast<int>(ret));
+  }
+  return ret;
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_GETRANDOM_H
diff --git a/libc/src/sys/random/linux/getrandom.cpp b/libc/src/sys/random/linux/getrandom.cpp
index 0b8471ed8b37..4a95bddfa428 100644
--- a/libc/src/sys/random/linux/getrandom.cpp
+++ b/libc/src/sys/random/linux/getrandom.cpp
@@ -8,24 +8,23 @@
 
 #include "src/sys/random/getrandom.h"
 
+#include "src/__support/OSUtil/linux/getrandom.h"
 #include "src/__support/OSUtil/syscall.h" // For internal syscall function.
 #include "src/__support/common.h"
-
+#include "src/__support/error_or.h"
 #include "src/__support/libc_errno.h"
 #include "src/__support/macros/config.h"
-#include <sys/syscall.h> // For syscall numbers.
 
 namespace LIBC_NAMESPACE_DECL {
 
 LLVM_LIBC_FUNCTION(ssize_t, getrandom,
                    (void *buf, size_t buflen, unsigned int flags)) {
-  ssize_t ret =
-      LIBC_NAMESPACE::syscall_impl<ssize_t>(SYS_getrandom, buf, buflen, flags);
-  if (ret < 0) {
-    libc_errno = static_cast<int>(-ret);
+  auto rand = internal::getrandom(buf, buflen, flags);
+  if (!rand.has_value()) {
+    libc_errno = static_cast<int>(rand.error());
     return -1;
   }
-  return ret;
+  return rand.value();
 }
 
 } // namespace LIBC_NAMESPACE_DECL
-- 
cgit v1.2.3


From dfe4d44d8de645d151d3483272c1c1f80c27ab31 Mon Sep 17 00:00:00 2001
From: Arthur Eubanks <aeubanks@google.com>
Date: Wed, 18 Jun 2025 11:00:13 -0700
Subject: Revert "[VPlan] Remove unnecessary DomTreeUpdater flush (NFC)."
 (#144758)

This reverts commit 2e337349f436d75af112c081df5ec683871cbcc8.

Causes breakages internally, will post reproducer later.
---
 llvm/lib/Transforms/Vectorize/VPlan.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp
index 4332332ef5cc..773a5a4a829c 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp
@@ -1015,6 +1015,8 @@ void VPlan::execute(VPTransformState *State) {
   for (VPBlockBase *Block : RPOT)
     Block->execute(State);
 
+  State->CFG.DTU.flush();
+
   VPBasicBlock *Header = vputils::getFirstLoopHeader(*this, State->VPDT);
   if (!Header)
     return;
-- 
cgit v1.2.3


From 071a6feabd7aeec2c1239719f50f6912cf94d00a Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 18 Jun 2025 19:02:17 +0100
Subject: [TTI] Remove PPC hasActiveVectorLength impl, simplify interface
 (NFC). (#142310)

PPCTTIImpl defines hasActiveVectorLength and also getVPMemoryOpCost, but
they appear unused (i.e. no changes to tests).

Remove them, as they complicate the interface for hasActiveVectorLength.
This simplifies the only use in LV as now no placeholder values need to
be passed.

PR: https://github.com/llvm/llvm-project/pull/142310
---
 llvm/include/llvm/Analysis/TargetTransformInfo.h   |  8 +--
 .../llvm/Analysis/TargetTransformInfoImpl.h        |  5 +-
 llvm/lib/Analysis/TargetTransformInfo.cpp          |  5 +-
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 82 ----------------------
 llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h   |  6 --
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp |  2 +-
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h   | 13 +---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    |  4 +-
 8 files changed, 11 insertions(+), 114 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 8f4ce80ada5e..9dc4eca82492 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1850,11 +1850,9 @@ public:
   /// \name Vector Predication Information
   /// @{
   /// Whether the target supports the %evl parameter of VP intrinsic efficiently
-  /// in hardware, for the given opcode and type/alignment. (see LLVM Language
-  /// Reference - "Vector Predication Intrinsics").
-  /// Use of %evl is discouraged when that is not the case.
-  LLVM_ABI bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                                      Align Alignment) const;
+  /// in hardware. (see LLVM Language Reference - "Vector Predication
+  /// Intrinsics"). Use of %evl is discouraged when that is not the case.
+  LLVM_ABI bool hasActiveVectorLength() const;
 
   /// Return true if sinking I's operands to the same basic block as I is
   /// profitable, e.g. because the operands can be folded into a target
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index a80b4c5179ba..d93375218394 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -1108,10 +1108,7 @@ public:
 
   virtual bool enableScalableVectorization() const { return false; }
 
-  virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                                     Align Alignment) const {
-    return false;
-  }
+  virtual bool hasActiveVectorLength() const { return false; }
 
   virtual bool isProfitableToSinkOperands(Instruction *I,
                                           SmallVectorImpl<Use *> &Ops) const {
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 2d053e55bdfa..d9cb11de9c09 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1454,9 +1454,8 @@ bool TargetTransformInfo::enableScalableVectorization() const {
   return TTIImpl->enableScalableVectorization();
 }
 
-bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                                                Align Alignment) const {
-  return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
+bool TargetTransformInfo::hasActiveVectorLength() const {
+  return TTIImpl->hasActiveVectorLength();
 }
 
 bool TargetTransformInfo::isProfitableToSinkOperands(
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
index f9e77f2abdca..cd9b226ca82d 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
@@ -1027,88 +1027,6 @@ bool PPCTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
   return false;
 }
 
-bool PPCTTIImpl::hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                                       Align Alignment) const {
-  // Only load and stores instructions can have variable vector length on Power.
-  if (Opcode != Instruction::Load && Opcode != Instruction::Store)
-    return false;
-  // Loads/stores with length instructions use bits 0-7 of the GPR operand and
-  // therefore cannot be used in 32-bit mode.
-  if ((!ST->hasP9Vector() && !ST->hasP10Vector()) || !ST->isPPC64())
-    return false;
-  if (isa<FixedVectorType>(DataType)) {
-    unsigned VecWidth = DataType->getPrimitiveSizeInBits();
-    return VecWidth == 128;
-  }
-  Type *ScalarTy = DataType->getScalarType();
-
-  if (ScalarTy->isPointerTy())
-    return true;
-
-  if (ScalarTy->isFloatTy() || ScalarTy->isDoubleTy())
-    return true;
-
-  if (!ScalarTy->isIntegerTy())
-    return false;
-
-  unsigned IntWidth = ScalarTy->getIntegerBitWidth();
-  return IntWidth == 8 || IntWidth == 16 || IntWidth == 32 || IntWidth == 64;
-}
-
-InstructionCost PPCTTIImpl::getVPMemoryOpCost(unsigned Opcode, Type *Src,
-                                              Align Alignment,
-                                              unsigned AddressSpace,
-                                              TTI::TargetCostKind CostKind,
-                                              const Instruction *I) const {
-  InstructionCost Cost = BaseT::getVPMemoryOpCost(Opcode, Src, Alignment,
-                                                  AddressSpace, CostKind, I);
-  if (TLI->getValueType(DL, Src, true) == MVT::Other)
-    return Cost;
-  // TODO: Handle other cost kinds.
-  if (CostKind != TTI::TCK_RecipThroughput)
-    return Cost;
-
-  assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
-         "Invalid Opcode");
-
-  auto *SrcVTy = dyn_cast<FixedVectorType>(Src);
-  assert(SrcVTy && "Expected a vector type for VP memory operations");
-
-  if (hasActiveVectorLength(Opcode, Src, Alignment)) {
-    std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcVTy);
-
-    InstructionCost CostFactor =
-        vectorCostAdjustmentFactor(Opcode, Src, nullptr);
-    if (!CostFactor.isValid())
-      return InstructionCost::getMax();
-
-    InstructionCost Cost = LT.first * CostFactor;
-    assert(Cost.isValid() && "Expected valid cost");
-
-    // On P9 but not on P10, if the op is misaligned then it will cause a
-    // pipeline flush. Otherwise the VSX masked memops cost the same as unmasked
-    // ones.
-    const Align DesiredAlignment(16);
-    if (Alignment >= DesiredAlignment || ST->getCPUDirective() != PPC::DIR_PWR9)
-      return Cost;
-
-    // Since alignment may be under estimated, we try to compute the probability
-    // that the actual address is aligned to the desired boundary. For example
-    // an 8-byte aligned load is assumed to be actually 16-byte aligned half the
-    // time, while a 4-byte aligned load has a 25% chance of being 16-byte
-    // aligned.
-    float AlignmentProb = ((float)Alignment.value()) / DesiredAlignment.value();
-    float MisalignmentProb = 1.0 - AlignmentProb;
-    return (MisalignmentProb * P9PipelineFlushEstimate) +
-           (AlignmentProb * Cost.getValue());
-  }
-
-  // Usually we should not get to this point, but the following is an attempt to
-  // model the cost of legalization. Currently we can only lower intrinsics with
-  // evl but no mask, on Power 9/10. Otherwise, we must scalarize.
-  return getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
-}
-
 bool PPCTTIImpl::supportsTailCallFor(const CallBase *CB) const {
   return TLI->supportsTailCallFor(CB);
 }
diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
index 8618f3064c18..bc5f7a4d06de 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
@@ -148,12 +148,6 @@ public:
                            const Function *Callee) const override;
   bool areTypesABICompatible(const Function *Caller, const Function *Callee,
                              const ArrayRef<Type *> &Types) const override;
-  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                             Align Alignment) const override;
-  InstructionCost
-  getVPMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment,
-                    unsigned AddressSpace, TTI::TargetCostKind CostKind,
-                    const Instruction *I = nullptr) const override;
   bool supportsTailCallFor(const CallBase *CB) const override;
 
 private:
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 46e30ce4c18a..63c5f17a8487 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -282,7 +282,7 @@ RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
   return TTI::TCC_Free;
 }
 
-bool RISCVTTIImpl::hasActiveVectorLength(unsigned, Type *DataTy, Align) const {
+bool RISCVTTIImpl::hasActiveVectorLength() const {
   return ST->hasVInstructions();
 }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index dd7e9f7709f8..75d377abb0e7 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -90,19 +90,12 @@ public:
 
   /// \name EVL Support for predicated vectorization.
   /// Whether the target supports the %evl parameter of VP intrinsic efficiently
-  /// in hardware, for the given opcode and type/alignment. (see LLVM Language
-  /// Reference - "Vector Predication Intrinsics",
+  /// in hardware. (see LLVM Language Reference - "Vector Predication
+  /// Intrinsics",
   /// https://llvm.org/docs/LangRef.html#vector-predication-intrinsics and
   /// "IR-level VP intrinsics",
   /// https://llvm.org/docs/Proposals/VectorPredication.html#ir-level-vp-intrinsics).
-  /// \param Opcode the opcode of the instruction checked for predicated version
-  /// support.
-  /// \param DataType the type of the instruction with the \p Opcode checked for
-  /// prediction support.
-  /// \param Alignment the alignment for memory access operation checked for
-  /// predicated version support.
-  bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
-                             Align Alignment) const override;
+  bool hasActiveVectorLength() const override;
 
   TargetTransformInfo::PopcntSupportKind
   getPopcntSupport(unsigned TyWidth) const override;
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 2f4416d2782e..3b16248f962b 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1377,11 +1377,9 @@ public:
     if (ForceTailFoldingStyle != TailFoldingStyle::DataWithEVL)
       return;
     // Override forced styles if needed.
-    // FIXME: use actual opcode/data type for analysis here.
     // FIXME: Investigate opportunity for fixed vector factor.
     bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
-                      TTI.hasActiveVectorLength(0, nullptr, Align()) &&
-                      !EnableVPlanNativePath;
+                      TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
     if (!EVLIsLegal) {
       // If for some reason EVL mode is unsupported, fallback to
       // DataWithoutLaneMask to try to vectorize the loop with folded tail
-- 
cgit v1.2.3


From 3f3526f36d23eac8d099e8e887a924c94000bbfa Mon Sep 17 00:00:00 2001
From: zhijian lin <zhijian@ca.ibm.com>
Date: Wed, 18 Jun 2025 14:15:30 -0400
Subject: [NFC][PowerPC] pre-commit running the update_llc_test_checks.py for
 all-atomics.ll,loop-comment.ll etc (#144411)

Run the update_llc_test_checks.py for all-atomics.ll,loop-comment.ll
,PR35812-neg-cmpxchg.ll (Pre-commit patch for the
https://github.com/llvm/llvm-project/pull/144089)
---
 llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll |   94 +-
 llvm/test/CodeGen/PowerPC/all-atomics.ll         | 1929 +++++++++++-----------
 llvm/test/CodeGen/PowerPC/loop-comment.ll        |   14 +-
 3 files changed, 1020 insertions(+), 1017 deletions(-)

diff --git a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
index 1a8dabc5ad71..dac17dc3225e 100644
--- a/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
+++ b/llvm/test/CodeGen/PowerPC/PR35812-neg-cmpxchg.ll
@@ -18,54 +18,54 @@ define signext i32 @main() nounwind {
 ; CHECK-NEXT:    sth 3, 46(1)
 ; CHECK-NEXT:    addi 3, 1, 46
 ; CHECK-NEXT:    lharx 4, 0, 3
-; CHECK-NEXT:    clrlwi  4, 4, 16
-; CHECK-NEXT:    cmplwi  4, 33059
-; CHECK-NEXT:    bne     0, .LBB0_4
-; CHECK-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
+; CHECK-NEXT:    clrlwi 4, 4, 16
+; CHECK-NEXT:    cmplwi 4, 33059
+; CHECK-NEXT:    bne 0, .LBB0_4
+; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore
 ; CHECK-NEXT:    sync
 ; CHECK-NEXT:    li 4, 234
-; CHECK-NEXT:    .p2align        5
-; CHECK-NEXT:  .LBB0_2:                                # %cmpxchg.trystore
-; CHECK-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB0_2: # %cmpxchg.trystore
+; CHECK-NEXT:    #
 ; CHECK-NEXT:    sthcx. 4, 0, 3
-; CHECK-NEXT:    beq     0, .LBB0_7
-; CHECK-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
-; CHECK-NEXT:                                          #   in Loop: Header=BB0_2 Depth=1
+; CHECK-NEXT:    beq 0, .LBB0_7
+; CHECK-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; CHECK-NEXT:    #
 ; CHECK-NEXT:    lharx 5, 0, 3
-; CHECK-NEXT:    clrlwi  5, 5, 16
-; CHECK-NEXT:    cmplwi  5, 33059
-; CHECK-NEXT:    beq     0, .LBB0_2
-; CHECK-NEXT:  .LBB0_4:                                # %cmpxchg.nostore
+; CHECK-NEXT:    clrlwi 5, 5, 16
+; CHECK-NEXT:    cmplwi 5, 33059
+; CHECK-NEXT:    beq 0, .LBB0_2
+; CHECK-NEXT:  .LBB0_4: # %cmpxchg.nostore
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    b .LBB0_8
-; CHECK-NEXT:  .LBB0_5:                                # %L.B0000
+; CHECK-NEXT:  .LBB0_5: # %L.B0000
 ; CHECK-NEXT:    lhz 3, 46(1)
-; CHECK-NEXT:    cmplwi  3, 234
-; CHECK-NEXT:    bne     0, .LBB0_9
-; CHECK-NEXT:  # %bb.6:                                # %L.B0001
+; CHECK-NEXT:    cmplwi 3, 234
+; CHECK-NEXT:    bne 0, .LBB0_9
+; CHECK-NEXT:  # %bb.6: # %L.B0001
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    bl puts
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    li 3, 0
 ; CHECK-NEXT:    b .LBB0_11
-; CHECK-NEXT:  .LBB0_7:                                # %cmpxchg.success
+; CHECK-NEXT:  .LBB0_7: # %cmpxchg.success
 ; CHECK-NEXT:    lwsync
 ; CHECK-NEXT:    b .LBB0_5
-; CHECK-NEXT:  .LBB0_8:                                # %L.B0003
+; CHECK-NEXT:  .LBB0_8: # %L.B0003
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    addi 3, 3, 16
 ; CHECK-NEXT:    b .LBB0_10
-; CHECK-NEXT:  .LBB0_9:                                # %L.B0005
+; CHECK-NEXT:  .LBB0_9: # %L.B0005
 ; CHECK-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-NEXT:    addi 3, 3, 64
-; CHECK-NEXT:  .LBB0_10:                               # %L.B0003
+; CHECK-NEXT:  .LBB0_10: # %L.B0003
 ; CHECK-NEXT:    bl puts
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    li 3, 1
-; CHECK-NEXT:  .LBB0_11:                               # %L.B0003
+; CHECK-NEXT:  .LBB0_11: # %L.B0003
 ; CHECK-NEXT:    addi 1, 1, 48
 ; CHECK-NEXT:    ld 0, 16(1)
 ; CHECK-NEXT:    mtlr 0
@@ -83,62 +83,62 @@ define signext i32 @main() nounwind {
 ; CHECK-P7-NEXT:    rlwinm 4, 4, 3, 27, 27
 ; CHECK-P7-NEXT:    lwarx 5, 0, 3
 ; CHECK-P7-NEXT:    srw 6, 5, 4
-; CHECK-P7-NEXT:    clrlwi  6, 6, 16
-; CHECK-P7-NEXT:    cmplwi  6, 33059
-; CHECK-P7-NEXT:    bne     0, .LBB0_4
-; CHECK-P7-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore
+; CHECK-P7-NEXT:    clrlwi 6, 6, 16
+; CHECK-P7-NEXT:    cmplwi 6, 33059
+; CHECK-P7-NEXT:    bne 0, .LBB0_4
+; CHECK-P7-NEXT:  # %bb.1: # %cmpxchg.fencedstore
 ; CHECK-P7-NEXT:    lis 6, 0
 ; CHECK-P7-NEXT:    li 7, 234
 ; CHECK-P7-NEXT:    sync
 ; CHECK-P7-NEXT:    ori 6, 6, 65535
 ; CHECK-P7-NEXT:    slw 7, 7, 4
 ; CHECK-P7-NEXT:    slw 6, 6, 4
-; CHECK-P7-NEXT:    not     6, 6
-; CHECK-P7-NEXT:    .p2align        4
-; CHECK-P7-NEXT:  .LBB0_2:                                # %cmpxchg.trystore
-; CHECK-P7-NEXT:                                        # =>This Inner Loop Header: Depth=1
+; CHECK-P7-NEXT:    not 6, 6
+; CHECK-P7-NEXT:    .p2align 4
+; CHECK-P7-NEXT:  .LBB0_2: # %cmpxchg.trystore
+; CHECK-P7-NEXT:    #
 ; CHECK-P7-NEXT:    and 5, 5, 6
 ; CHECK-P7-NEXT:    or 5, 5, 7
 ; CHECK-P7-NEXT:    stwcx. 5, 0, 3
-; CHECK-P7-NEXT:    beq     0, .LBB0_7
-; CHECK-P7-NEXT:  # %bb.3:                                # %cmpxchg.releasedload
-; CHECK-P7-NEXT:                                        #   in Loop: Header=BB0_2 Depth=1
+; CHECK-P7-NEXT:    beq 0, .LBB0_7
+; CHECK-P7-NEXT:  # %bb.3: # %cmpxchg.releasedload
+; CHECK-P7-NEXT:    #
 ; CHECK-P7-NEXT:    lwarx 5, 0, 3
 ; CHECK-P7-NEXT:    srw 8, 5, 4
-; CHECK-P7-NEXT:    clrlwi  8, 8, 16
-; CHECK-P7-NEXT:    cmplwi  8, 33059
-; CHECK-P7-NEXT:    beq     0, .LBB0_2
-; CHECK-P7-NEXT:  .LBB0_4:                                # %cmpxchg.nostore
+; CHECK-P7-NEXT:    clrlwi 8, 8, 16
+; CHECK-P7-NEXT:    cmplwi 8, 33059
+; CHECK-P7-NEXT:    beq 0, .LBB0_2
+; CHECK-P7-NEXT:  .LBB0_4: # %cmpxchg.nostore
 ; CHECK-P7-NEXT:    lwsync
 ; CHECK-P7-NEXT:    b .LBB0_8
-; CHECK-P7-NEXT:  .LBB0_5:                                # %L.B0000
+; CHECK-P7-NEXT:  .LBB0_5: # %L.B0000
 ; CHECK-P7-NEXT:    lhz 3, 46(1)
-; CHECK-P7-NEXT:    cmplwi  3, 234
-; CHECK-P7-NEXT:    bne     0, .LBB0_9
-; CHECK-P7-NEXT:  # %bb.6:                                # %L.B0001
+; CHECK-P7-NEXT:    cmplwi 3, 234
+; CHECK-P7-NEXT:    bne 0, .LBB0_9
+; CHECK-P7-NEXT:  # %bb.6: # %L.B0001
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    bl puts
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    li 3, 0
 ; CHECK-P7-NEXT:    b .LBB0_11
-; CHECK-P7-NEXT:  .LBB0_7:                                # %cmpxchg.success
+; CHECK-P7-NEXT:  .LBB0_7: # %cmpxchg.success
 ; CHECK-P7-NEXT:    lwsync
 ; CHECK-P7-NEXT:    b .LBB0_5
-; CHECK-P7-NEXT:  .LBB0_8:                                # %L.B0003
+; CHECK-P7-NEXT:  .LBB0_8: # %L.B0003
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    addi 3, 3, 16
 ; CHECK-P7-NEXT:    b .LBB0_10
-; CHECK-P7-NEXT:  .LBB0_9:                                # %L.B0005
+; CHECK-P7-NEXT:  .LBB0_9: # %L.B0005
 ; CHECK-P7-NEXT:    addis 3, 2, .L_MergedGlobals@toc@ha
 ; CHECK-P7-NEXT:    addi 3, 3, .L_MergedGlobals@toc@l
 ; CHECK-P7-NEXT:    addi 3, 3, 64
-; CHECK-P7-NEXT:  .LBB0_10:                               # %L.B0003
+; CHECK-P7-NEXT:  .LBB0_10: # %L.B0003
 ; CHECK-P7-NEXT:    bl puts
 ; CHECK-P7-NEXT:    nop
 ; CHECK-P7-NEXT:    li 3, 1
-; CHECK-P7-NEXT:  .LBB0_11:                               # %L.B0003
+; CHECK-P7-NEXT:  .LBB0_11: # %L.B0003
 ; CHECK-P7-NEXT:    addi 1, 1, 48
 ; CHECK-P7-NEXT:    ld 0, 16(1)
 ; CHECK-P7-NEXT:    mtlr 0
diff --git a/llvm/test/CodeGen/PowerPC/all-atomics.ll b/llvm/test/CodeGen/PowerPC/all-atomics.ll
index 67cee358882f..5e14fbbb6ad6 100644
--- a/llvm/test/CodeGen/PowerPC/all-atomics.ll
+++ b/llvm/test/CodeGen/PowerPC/all-atomics.ll
@@ -4336,959 +4336,959 @@ entry:
 define dso_local void @test_compare_and_swap() local_unnamed_addr #0 {
 ; CHECK-LABEL: test_compare_and_swap:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:   addis 4, 2, sc@toc@ha
-; CHECK-NEXT:   addis 3, 2, uc@toc@ha
-; CHECK-NEXT:   std 27, -40(1)                          # 8-byte Folded Spill
-; CHECK-NEXT:   std 28, -32(1)                          # 8-byte Folded Spill
-; CHECK-NEXT:   std 29, -24(1)                          # 8-byte Folded Spill
-; CHECK-NEXT:   std 30, -16(1)                          # 8-byte Folded Spill
-; CHECK-NEXT:   addi 6, 4, sc@toc@l
-; CHECK-NEXT:   lbz 7, uc@toc@l(3)
-; CHECK-NEXT:   lbz 8, sc@toc@l(4)
-; CHECK-NEXT:   lbarx 5, 0, 6
-; CHECK-NEXT:   clrlwi  9, 5, 24
-; CHECK-NEXT:   cmplw   9, 7
-; CHECK-NEXT:   bne     0, .LBB3_4
-; CHECK-NEXT: # %bb.1:                                # %cmpxchg.fencedstore276
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_2:                                # %cmpxchg.trystore275
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stbcx. 8, 0, 6
-; CHECK-NEXT:   beq     0, .LBB3_4
-; CHECK-NEXT: # %bb.3:                                # %cmpxchg.releasedload274
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_2 Depth=1
-; CHECK-NEXT:   lbarx 5, 0, 6
-; CHECK-NEXT:   clrlwi  9, 5, 24
-; CHECK-NEXT:   cmplw   9, 7
-; CHECK-NEXT:   beq     0, .LBB3_2
-; CHECK-NEXT: .LBB3_4:                                # %cmpxchg.nostore272
-; CHECK-NEXT:   addi 7, 3, uc@toc@l
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   stb 5, sc@toc@l(4)
-; CHECK-NEXT:   lbz 9, uc@toc@l(3)
-; CHECK-NEXT:   lbarx 8, 0, 7
-; CHECK-NEXT:   clrlwi  10, 8, 24
-; CHECK-NEXT:   cmplw   10, 9
-; CHECK-NEXT:   bne     0, .LBB3_8
-; CHECK-NEXT: # %bb.5:                                # %cmpxchg.fencedstore257
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   clrlwi  5, 5, 24
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_6:                                # %cmpxchg.trystore256
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stbcx. 5, 0, 7
-; CHECK-NEXT:   beq     0, .LBB3_8
-; CHECK-NEXT: # %bb.7:                                # %cmpxchg.releasedload255
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_6 Depth=1
-; CHECK-NEXT:   lbarx 8, 0, 7
-; CHECK-NEXT:   clrlwi  10, 8, 24
-; CHECK-NEXT:   cmplw   10, 9
-; CHECK-NEXT:   beq     0, .LBB3_6
-; CHECK-NEXT: .LBB3_8:                                # %cmpxchg.nostore253
-; CHECK-NEXT:   addis 5, 2, ss@toc@ha
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   stb 8, uc@toc@l(3)
-; CHECK-NEXT:   clrlwi  10, 8, 24
-; CHECK-NEXT:   lbz 11, sc@toc@l(4)
-; CHECK-NEXT:   addi 8, 5, ss@toc@l
-; CHECK-NEXT:   lharx 9, 0, 8
-; CHECK-NEXT:   clrlwi  12, 9, 16
-; CHECK-NEXT:   cmplw   12, 10
-; CHECK-NEXT:   bne     0, .LBB3_12
-; CHECK-NEXT: # %bb.9:                                # %cmpxchg.fencedstore238
-; CHECK-NEXT:   extsb 11, 11
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   clrlwi  11, 11, 16
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_10:                               # %cmpxchg.trystore237
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   sthcx. 11, 0, 8
-; CHECK-NEXT:   beq     0, .LBB3_12
-; CHECK-NEXT: # %bb.11:                               # %cmpxchg.releasedload236
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_10 Depth=1
-; CHECK-NEXT:   lharx 9, 0, 8
-; CHECK-NEXT:   clrlwi  12, 9, 16
-; CHECK-NEXT:   cmplw   12, 10
-; CHECK-NEXT:   beq     0, .LBB3_10
-; CHECK-NEXT: .LBB3_12:                               # %cmpxchg.nostore234
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   sth 9, ss@toc@l(5)
-; CHECK-NEXT:   addis 5, 2, us@toc@ha
-; CHECK-NEXT:   lbz 11, uc@toc@l(3)
-; CHECK-NEXT:   lbz 12, sc@toc@l(4)
-; CHECK-NEXT:   addi 9, 5, us@toc@l
-; CHECK-NEXT:   lharx 10, 0, 9
-; CHECK-NEXT:   clrlwi  0, 10, 16
-; CHECK-NEXT:   cmplw   0, 11
-; CHECK-NEXT:   bne     0, .LBB3_16
-; CHECK-NEXT: # %bb.13:                               # %cmpxchg.fencedstore219
-; CHECK-NEXT:   extsb 12, 12
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   clrlwi  12, 12, 16
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_14:                               # %cmpxchg.trystore218
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   sthcx. 12, 0, 9
-; CHECK-NEXT:   beq     0, .LBB3_16
-; CHECK-NEXT: # %bb.15:                               # %cmpxchg.releasedload217
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_14 Depth=1
-; CHECK-NEXT:   lharx 10, 0, 9
-; CHECK-NEXT:   clrlwi  0, 10, 16
-; CHECK-NEXT:   cmplw   0, 11
-; CHECK-NEXT:   beq     0, .LBB3_14
-; CHECK-NEXT: .LBB3_16:                               # %cmpxchg.nostore215
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   sth 10, us@toc@l(5)
-; CHECK-NEXT:   addis 5, 2, si@toc@ha
-; CHECK-NEXT:   lbz 12, uc@toc@l(3)
-; CHECK-NEXT:   lbz 0, sc@toc@l(4)
-; CHECK-NEXT:   addi 10, 5, si@toc@l
-; CHECK-NEXT:   lwarx 11, 0, 10
-; CHECK-NEXT:   cmplw   11, 12
-; CHECK-NEXT:   bne     0, .LBB3_20
-; CHECK-NEXT: # %bb.17:                               # %cmpxchg.fencedstore200
-; CHECK-NEXT:   extsb 0, 0
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_18:                               # %cmpxchg.trystore199
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stwcx. 0, 0, 10
-; CHECK-NEXT:   beq     0, .LBB3_20
-; CHECK-NEXT: # %bb.19:                               # %cmpxchg.releasedload198
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_18 Depth=1
-; CHECK-NEXT:   lwarx 11, 0, 10
-; CHECK-NEXT:   cmplw   11, 12
-; CHECK-NEXT:   beq     0, .LBB3_18
-; CHECK-NEXT: .LBB3_20:                               # %cmpxchg.nostore196
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   stw 11, si@toc@l(5)
-; CHECK-NEXT:   addis 5, 2, ui@toc@ha
-; CHECK-NEXT:   lbz 0, uc@toc@l(3)
-; CHECK-NEXT:   lbz 30, sc@toc@l(4)
-; CHECK-NEXT:   addi 11, 5, ui@toc@l
-; CHECK-NEXT:   lwarx 12, 0, 11
-; CHECK-NEXT:   cmplw   12, 0
-; CHECK-NEXT:   bne     0, .LBB3_24
-; CHECK-NEXT: # %bb.21:                               # %cmpxchg.fencedstore181
-; CHECK-NEXT:   extsb 30, 30
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_22:                               # %cmpxchg.trystore180
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stwcx. 30, 0, 11
-; CHECK-NEXT:   beq     0, .LBB3_24
-; CHECK-NEXT: # %bb.23:                               # %cmpxchg.releasedload179
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_22 Depth=1
-; CHECK-NEXT:   lwarx 12, 0, 11
-; CHECK-NEXT:   cmplw   12, 0
-; CHECK-NEXT:   beq     0, .LBB3_22
-; CHECK-NEXT: .LBB3_24:                               # %cmpxchg.nostore177
-; CHECK-NEXT:   addis 30, 2, sll@toc@ha
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   stw 12, ui@toc@l(5)
-; CHECK-NEXT:   lbz 29, uc@toc@l(3)
-; CHECK-NEXT:   lbz 28, sc@toc@l(4)
-; CHECK-NEXT:   addi 12, 30, sll@toc@l
-; CHECK-NEXT:   ldarx 0, 0, 12
-; CHECK-NEXT:   cmpld   0, 29
-; CHECK-NEXT:   bne     0, .LBB3_28
-; CHECK-NEXT: # %bb.25:                               # %cmpxchg.fencedstore162
-; CHECK-NEXT:   extsb 28, 28
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_26:                               # %cmpxchg.trystore161
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stdcx. 28, 0, 12
-; CHECK-NEXT:   beq     0, .LBB3_28
-; CHECK-NEXT: # %bb.27:                               # %cmpxchg.releasedload160
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_26 Depth=1
-; CHECK-NEXT:   ldarx 0, 0, 12
-; CHECK-NEXT:   cmpld   0, 29
-; CHECK-NEXT:   beq     0, .LBB3_26
-; CHECK-NEXT: .LBB3_28:                               # %cmpxchg.nostore158
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   std 0, sll@toc@l(30)
-; CHECK-NEXT:   addis 30, 2, ull@toc@ha
-; CHECK-NEXT:   lbz 28, uc@toc@l(3)
-; CHECK-NEXT:   lbz 27, sc@toc@l(4)
-; CHECK-NEXT:   addi 0, 30, ull@toc@l
-; CHECK-NEXT:   ldarx 29, 0, 0
-; CHECK-NEXT:   cmpld   29, 28
-; CHECK-NEXT:   bne     0, .LBB3_32
-; CHECK-NEXT: # %bb.29:                               # %cmpxchg.fencedstore143
-; CHECK-NEXT:   extsb 27, 27
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_30:                               # %cmpxchg.trystore142
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stdcx. 27, 0, 0
-; CHECK-NEXT:   beq     0, .LBB3_32
-; CHECK-NEXT: # %bb.31:                               # %cmpxchg.releasedload141
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_30 Depth=1
-; CHECK-NEXT:   ldarx 29, 0, 0
-; CHECK-NEXT:   cmpld   29, 28
-; CHECK-NEXT:   beq     0, .LBB3_30
-; CHECK-NEXT: .LBB3_32:                               # %cmpxchg.nostore139
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   std 29, ull@toc@l(30)
-; CHECK-NEXT:   lbz 30, uc@toc@l(3)
-; CHECK-NEXT:   lbz 29, sc@toc@l(4)
-; CHECK-NEXT:   lbarx 28, 0, 6
-; CHECK-NEXT:   clrlwi  28, 28, 24
-; CHECK-NEXT:   cmplw   28, 30
-; CHECK-NEXT:   bne     0, .LBB3_36
-; CHECK-NEXT: # %bb.33:                               # %cmpxchg.fencedstore124
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_34:                               # %cmpxchg.trystore123
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stbcx. 29, 0, 6
-; CHECK-NEXT:   beq     0, .LBB3_37
-; CHECK-NEXT: # %bb.35:                               # %cmpxchg.releasedload122
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_34 Depth=1
-; CHECK-NEXT:   lbarx 28, 0, 6
-; CHECK-NEXT:   clrlwi  28, 28, 24
-; CHECK-NEXT:   cmplw   28, 30
-; CHECK-NEXT:   beq     0, .LBB3_34
-; CHECK-NEXT: .LBB3_36:                               # %cmpxchg.nostore120
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_38
-; CHECK-NEXT: .LBB3_37:                               # %cmpxchg.success121
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_38:                               # %cmpxchg.end118
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 30, 1
-; CHECK-NEXT:   isel 6, 30, 6, 20
-; CHECK-NEXT:   lbz 30, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   lbarx 29, 0, 7
-; CHECK-NEXT:   clrlwi  29, 29, 24
-; CHECK-NEXT:   cmplw   29, 6
-; CHECK-NEXT:   bne     0, .LBB3_42
-; CHECK-NEXT: # %bb.39:                               # %cmpxchg.fencedstore105
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_40:                               # %cmpxchg.trystore104
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stbcx. 30, 0, 7
-; CHECK-NEXT:   beq     0, .LBB3_43
-; CHECK-NEXT: # %bb.41:                               # %cmpxchg.releasedload103
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_40 Depth=1
-; CHECK-NEXT:   lbarx 29, 0, 7
-; CHECK-NEXT:   clrlwi  29, 29, 24
-; CHECK-NEXT:   cmplw   29, 6
-; CHECK-NEXT:   beq     0, .LBB3_40
-; CHECK-NEXT: .LBB3_42:                               # %cmpxchg.nostore101
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_44
-; CHECK-NEXT: .LBB3_43:                               # %cmpxchg.success102
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_44:                               # %cmpxchg.end99
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   lbz 7, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   lharx 30, 0, 8
-; CHECK-NEXT:   clrlwi  30, 30, 16
-; CHECK-NEXT:   cmplw   30, 6
-; CHECK-NEXT:   bne     0, .LBB3_48
-; CHECK-NEXT: # %bb.45:                               # %cmpxchg.fencedstore86
-; CHECK-NEXT:   extsb 7, 7
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   clrlwi  7, 7, 16
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_46:                               # %cmpxchg.trystore85
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   sthcx. 7, 0, 8
-; CHECK-NEXT:   beq     0, .LBB3_49
-; CHECK-NEXT: # %bb.47:                               # %cmpxchg.releasedload84
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_46 Depth=1
-; CHECK-NEXT:   lharx 30, 0, 8
-; CHECK-NEXT:   clrlwi  30, 30, 16
-; CHECK-NEXT:   cmplw   30, 6
-; CHECK-NEXT:   beq     0, .LBB3_46
-; CHECK-NEXT: .LBB3_48:                               # %cmpxchg.nostore82
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_50
-; CHECK-NEXT: .LBB3_49:                               # %cmpxchg.success83
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_50:                               # %cmpxchg.end80
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   lbz 7, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   lharx 8, 0, 9
-; CHECK-NEXT:   clrlwi  8, 8, 16
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   bne     0, .LBB3_54
-; CHECK-NEXT: # %bb.51:                               # %cmpxchg.fencedstore67
-; CHECK-NEXT:   extsb 7, 7
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   clrlwi  7, 7, 16
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_52:                               # %cmpxchg.trystore66
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   sthcx. 7, 0, 9
-; CHECK-NEXT:   beq     0, .LBB3_55
-; CHECK-NEXT: # %bb.53:                               # %cmpxchg.releasedload65
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_52 Depth=1
-; CHECK-NEXT:   lharx 8, 0, 9
-; CHECK-NEXT:   clrlwi  8, 8, 16
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   beq     0, .LBB3_52
-; CHECK-NEXT: .LBB3_54:                               # %cmpxchg.nostore63
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_56
-; CHECK-NEXT: .LBB3_55:                               # %cmpxchg.success64
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_56:                               # %cmpxchg.end61
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   lbz 7, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   lwarx 8, 0, 10
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   bne     0, .LBB3_60
-; CHECK-NEXT: # %bb.57:                               # %cmpxchg.fencedstore48
-; CHECK-NEXT:   extsb 7, 7
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_58:                               # %cmpxchg.trystore47
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stwcx. 7, 0, 10
-; CHECK-NEXT:   beq     0, .LBB3_61
-; CHECK-NEXT: # %bb.59:                               # %cmpxchg.releasedload46
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_58 Depth=1
-; CHECK-NEXT:   lwarx 8, 0, 10
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   beq     0, .LBB3_58
-; CHECK-NEXT: .LBB3_60:                               # %cmpxchg.nostore44
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_62
-; CHECK-NEXT: .LBB3_61:                               # %cmpxchg.success45
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_62:                               # %cmpxchg.end42
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   lbz 7, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   lwarx 8, 0, 11
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   bne     0, .LBB3_66
-; CHECK-NEXT: # %bb.63:                               # %cmpxchg.fencedstore29
-; CHECK-NEXT:   extsb 7, 7
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_64:                               # %cmpxchg.trystore28
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stwcx. 7, 0, 11
-; CHECK-NEXT:   beq     0, .LBB3_67
-; CHECK-NEXT: # %bb.65:                               # %cmpxchg.releasedload27
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_64 Depth=1
-; CHECK-NEXT:   lwarx 8, 0, 11
-; CHECK-NEXT:   cmplw   8, 6
-; CHECK-NEXT:   beq     0, .LBB3_64
-; CHECK-NEXT: .LBB3_66:                               # %cmpxchg.nostore25
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_68
-; CHECK-NEXT: .LBB3_67:                               # %cmpxchg.success26
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_68:                               # %cmpxchg.end23
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   lbz 7, sc@toc@l(4)
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   lbz 6, uc@toc@l(3)
-; CHECK-NEXT:   ldarx 8, 0, 12
-; CHECK-NEXT:   cmpld   8, 6
-; CHECK-NEXT:   bne     0, .LBB3_72
-; CHECK-NEXT: # %bb.69:                               # %cmpxchg.fencedstore10
-; CHECK-NEXT:   extsb 7, 7
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_70:                               # %cmpxchg.trystore9
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stdcx. 7, 0, 12
-; CHECK-NEXT:   beq     0, .LBB3_73
-; CHECK-NEXT: # %bb.71:                               # %cmpxchg.releasedload8
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_70 Depth=1
-; CHECK-NEXT:   ldarx 8, 0, 12
-; CHECK-NEXT:   cmpld   8, 6
-; CHECK-NEXT:   beq     0, .LBB3_70
-; CHECK-NEXT: .LBB3_72:                               # %cmpxchg.nostore6
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_74
-; CHECK-NEXT: .LBB3_73:                               # %cmpxchg.success7
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_74:                               # %cmpxchg.end4
-; CHECK-NEXT:   li 6, 0
-; CHECK-NEXT:   li 7, 1
-; CHECK-NEXT:   lbz 3, uc@toc@l(3)
-; CHECK-NEXT:   lbz 4, sc@toc@l(4)
-; CHECK-NEXT:   isel 6, 7, 6, 20
-; CHECK-NEXT:   stw 6, ui@toc@l(5)
-; CHECK-NEXT:   ldarx 6, 0, 0
-; CHECK-NEXT:   cmpld   6, 3
-; CHECK-NEXT:   bne     0, .LBB3_78
-; CHECK-NEXT: # %bb.75:                               # %cmpxchg.fencedstore
-; CHECK-NEXT:   extsb 4, 4
-; CHECK-NEXT:   sync
-; CHECK-NEXT:   .p2align        5
-; CHECK-NEXT: .LBB3_76:                               # %cmpxchg.trystore
-; CHECK-NEXT:                                         # =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:   stdcx. 4, 0, 0
-; CHECK-NEXT:   beq     0, .LBB3_79
-; CHECK-NEXT: # %bb.77:                               # %cmpxchg.releasedload
-; CHECK-NEXT:                                         #   in Loop: Header=BB3_76 Depth=1
-; CHECK-NEXT:   ldarx 6, 0, 0
-; CHECK-NEXT:   cmpld   6, 3
-; CHECK-NEXT:   beq     0, .LBB3_76
-; CHECK-NEXT: .LBB3_78:                               # %cmpxchg.nostore
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT:   b .LBB3_80
-; CHECK-NEXT: .LBB3_79:                               # %cmpxchg.success
-; CHECK-NEXT:   lwsync
-; CHECK-NEXT:   creqv 20, 20, 20
-; CHECK-NEXT: .LBB3_80:                               # %cmpxchg.end
-; CHECK-NEXT:   li 3, 0
-; CHECK-NEXT:   li 4, 1
-; CHECK-NEXT:   ld 30, -16(1)                           # 8-byte Folded Reload
-; CHECK-NEXT:   ld 29, -24(1)                           # 8-byte Folded Reload
-; CHECK-NEXT:   ld 28, -32(1)                           # 8-byte Folded Reload
-; CHECK-NEXT:   ld 27, -40(1)                           # 8-byte Folded Reload
-; CHECK-NEXT:   isel 3, 4, 3, 20
-; CHECK-NEXT:   stw 3, ui@toc@l(5)
-; CHECK-NEXT:   blr
+; CHECK-NEXT:    addis 4, 2, sc@toc@ha
+; CHECK-NEXT:    addis 3, 2, uc@toc@ha
+; CHECK-NEXT:    std 27, -40(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 28, -32(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 29, -24(1) # 8-byte Folded Spill
+; CHECK-NEXT:    std 30, -16(1) # 8-byte Folded Spill
+; CHECK-NEXT:    addi 6, 4, sc@toc@l
+; CHECK-NEXT:    lbz 7, uc@toc@l(3)
+; CHECK-NEXT:    lbz 8, sc@toc@l(4)
+; CHECK-NEXT:    lbarx 5, 0, 6
+; CHECK-NEXT:    clrlwi 9, 5, 24
+; CHECK-NEXT:    cmplw 9, 7
+; CHECK-NEXT:    bne 0, .LBB3_4
+; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore276
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_2: # %cmpxchg.trystore275
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stbcx. 8, 0, 6
+; CHECK-NEXT:    beq 0, .LBB3_4
+; CHECK-NEXT:  # %bb.3: # %cmpxchg.releasedload274
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lbarx 5, 0, 6
+; CHECK-NEXT:    clrlwi 9, 5, 24
+; CHECK-NEXT:    cmplw 9, 7
+; CHECK-NEXT:    beq 0, .LBB3_2
+; CHECK-NEXT:  .LBB3_4: # %cmpxchg.nostore272
+; CHECK-NEXT:    addi 7, 3, uc@toc@l
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    stb 5, sc@toc@l(4)
+; CHECK-NEXT:    lbz 9, uc@toc@l(3)
+; CHECK-NEXT:    lbarx 8, 0, 7
+; CHECK-NEXT:    clrlwi 10, 8, 24
+; CHECK-NEXT:    cmplw 10, 9
+; CHECK-NEXT:    bne 0, .LBB3_8
+; CHECK-NEXT:  # %bb.5: # %cmpxchg.fencedstore257
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    clrlwi 5, 5, 24
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_6: # %cmpxchg.trystore256
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stbcx. 5, 0, 7
+; CHECK-NEXT:    beq 0, .LBB3_8
+; CHECK-NEXT:  # %bb.7: # %cmpxchg.releasedload255
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lbarx 8, 0, 7
+; CHECK-NEXT:    clrlwi 10, 8, 24
+; CHECK-NEXT:    cmplw 10, 9
+; CHECK-NEXT:    beq 0, .LBB3_6
+; CHECK-NEXT:  .LBB3_8: # %cmpxchg.nostore253
+; CHECK-NEXT:    addis 5, 2, ss@toc@ha
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    stb 8, uc@toc@l(3)
+; CHECK-NEXT:    clrlwi 10, 8, 24
+; CHECK-NEXT:    lbz 11, sc@toc@l(4)
+; CHECK-NEXT:    addi 8, 5, ss@toc@l
+; CHECK-NEXT:    lharx 9, 0, 8
+; CHECK-NEXT:    clrlwi 12, 9, 16
+; CHECK-NEXT:    cmplw 12, 10
+; CHECK-NEXT:    bne 0, .LBB3_12
+; CHECK-NEXT:  # %bb.9: # %cmpxchg.fencedstore238
+; CHECK-NEXT:    extsb 11, 11
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    clrlwi 11, 11, 16
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_10: # %cmpxchg.trystore237
+; CHECK-NEXT:    #
+; CHECK-NEXT:    sthcx. 11, 0, 8
+; CHECK-NEXT:    beq 0, .LBB3_12
+; CHECK-NEXT:  # %bb.11: # %cmpxchg.releasedload236
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lharx 9, 0, 8
+; CHECK-NEXT:    clrlwi 12, 9, 16
+; CHECK-NEXT:    cmplw 12, 10
+; CHECK-NEXT:    beq 0, .LBB3_10
+; CHECK-NEXT:  .LBB3_12: # %cmpxchg.nostore234
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    sth 9, ss@toc@l(5)
+; CHECK-NEXT:    addis 5, 2, us@toc@ha
+; CHECK-NEXT:    lbz 11, uc@toc@l(3)
+; CHECK-NEXT:    lbz 12, sc@toc@l(4)
+; CHECK-NEXT:    addi 9, 5, us@toc@l
+; CHECK-NEXT:    lharx 10, 0, 9
+; CHECK-NEXT:    clrlwi 0, 10, 16
+; CHECK-NEXT:    cmplw 0, 11
+; CHECK-NEXT:    bne 0, .LBB3_16
+; CHECK-NEXT:  # %bb.13: # %cmpxchg.fencedstore219
+; CHECK-NEXT:    extsb 12, 12
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    clrlwi 12, 12, 16
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_14: # %cmpxchg.trystore218
+; CHECK-NEXT:    #
+; CHECK-NEXT:    sthcx. 12, 0, 9
+; CHECK-NEXT:    beq 0, .LBB3_16
+; CHECK-NEXT:  # %bb.15: # %cmpxchg.releasedload217
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lharx 10, 0, 9
+; CHECK-NEXT:    clrlwi 0, 10, 16
+; CHECK-NEXT:    cmplw 0, 11
+; CHECK-NEXT:    beq 0, .LBB3_14
+; CHECK-NEXT:  .LBB3_16: # %cmpxchg.nostore215
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    sth 10, us@toc@l(5)
+; CHECK-NEXT:    addis 5, 2, si@toc@ha
+; CHECK-NEXT:    lbz 12, uc@toc@l(3)
+; CHECK-NEXT:    lbz 0, sc@toc@l(4)
+; CHECK-NEXT:    addi 10, 5, si@toc@l
+; CHECK-NEXT:    lwarx 11, 0, 10
+; CHECK-NEXT:    cmplw 11, 12
+; CHECK-NEXT:    bne 0, .LBB3_20
+; CHECK-NEXT:  # %bb.17: # %cmpxchg.fencedstore200
+; CHECK-NEXT:    extsb 0, 0
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_18: # %cmpxchg.trystore199
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stwcx. 0, 0, 10
+; CHECK-NEXT:    beq 0, .LBB3_20
+; CHECK-NEXT:  # %bb.19: # %cmpxchg.releasedload198
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lwarx 11, 0, 10
+; CHECK-NEXT:    cmplw 11, 12
+; CHECK-NEXT:    beq 0, .LBB3_18
+; CHECK-NEXT:  .LBB3_20: # %cmpxchg.nostore196
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    stw 11, si@toc@l(5)
+; CHECK-NEXT:    addis 5, 2, ui@toc@ha
+; CHECK-NEXT:    lbz 0, uc@toc@l(3)
+; CHECK-NEXT:    lbz 30, sc@toc@l(4)
+; CHECK-NEXT:    addi 11, 5, ui@toc@l
+; CHECK-NEXT:    lwarx 12, 0, 11
+; CHECK-NEXT:    cmplw 12, 0
+; CHECK-NEXT:    bne 0, .LBB3_24
+; CHECK-NEXT:  # %bb.21: # %cmpxchg.fencedstore181
+; CHECK-NEXT:    extsb 30, 30
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_22: # %cmpxchg.trystore180
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stwcx. 30, 0, 11
+; CHECK-NEXT:    beq 0, .LBB3_24
+; CHECK-NEXT:  # %bb.23: # %cmpxchg.releasedload179
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lwarx 12, 0, 11
+; CHECK-NEXT:    cmplw 12, 0
+; CHECK-NEXT:    beq 0, .LBB3_22
+; CHECK-NEXT:  .LBB3_24: # %cmpxchg.nostore177
+; CHECK-NEXT:    addis 30, 2, sll@toc@ha
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    stw 12, ui@toc@l(5)
+; CHECK-NEXT:    lbz 29, uc@toc@l(3)
+; CHECK-NEXT:    lbz 28, sc@toc@l(4)
+; CHECK-NEXT:    addi 12, 30, sll@toc@l
+; CHECK-NEXT:    ldarx 0, 0, 12
+; CHECK-NEXT:    cmpld 0, 29
+; CHECK-NEXT:    bne 0, .LBB3_28
+; CHECK-NEXT:  # %bb.25: # %cmpxchg.fencedstore162
+; CHECK-NEXT:    extsb 28, 28
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_26: # %cmpxchg.trystore161
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stdcx. 28, 0, 12
+; CHECK-NEXT:    beq 0, .LBB3_28
+; CHECK-NEXT:  # %bb.27: # %cmpxchg.releasedload160
+; CHECK-NEXT:    #
+; CHECK-NEXT:    ldarx 0, 0, 12
+; CHECK-NEXT:    cmpld 0, 29
+; CHECK-NEXT:    beq 0, .LBB3_26
+; CHECK-NEXT:  .LBB3_28: # %cmpxchg.nostore158
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    std 0, sll@toc@l(30)
+; CHECK-NEXT:    addis 30, 2, ull@toc@ha
+; CHECK-NEXT:    lbz 28, uc@toc@l(3)
+; CHECK-NEXT:    lbz 27, sc@toc@l(4)
+; CHECK-NEXT:    addi 0, 30, ull@toc@l
+; CHECK-NEXT:    ldarx 29, 0, 0
+; CHECK-NEXT:    cmpld 29, 28
+; CHECK-NEXT:    bne 0, .LBB3_32
+; CHECK-NEXT:  # %bb.29: # %cmpxchg.fencedstore143
+; CHECK-NEXT:    extsb 27, 27
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_30: # %cmpxchg.trystore142
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stdcx. 27, 0, 0
+; CHECK-NEXT:    beq 0, .LBB3_32
+; CHECK-NEXT:  # %bb.31: # %cmpxchg.releasedload141
+; CHECK-NEXT:    #
+; CHECK-NEXT:    ldarx 29, 0, 0
+; CHECK-NEXT:    cmpld 29, 28
+; CHECK-NEXT:    beq 0, .LBB3_30
+; CHECK-NEXT:  .LBB3_32: # %cmpxchg.nostore139
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    std 29, ull@toc@l(30)
+; CHECK-NEXT:    lbz 30, uc@toc@l(3)
+; CHECK-NEXT:    lbz 29, sc@toc@l(4)
+; CHECK-NEXT:    lbarx 28, 0, 6
+; CHECK-NEXT:    clrlwi 28, 28, 24
+; CHECK-NEXT:    cmplw 28, 30
+; CHECK-NEXT:    bne 0, .LBB3_36
+; CHECK-NEXT:  # %bb.33: # %cmpxchg.fencedstore124
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_34: # %cmpxchg.trystore123
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stbcx. 29, 0, 6
+; CHECK-NEXT:    beq 0, .LBB3_37
+; CHECK-NEXT:  # %bb.35: # %cmpxchg.releasedload122
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lbarx 28, 0, 6
+; CHECK-NEXT:    clrlwi 28, 28, 24
+; CHECK-NEXT:    cmplw 28, 30
+; CHECK-NEXT:    beq 0, .LBB3_34
+; CHECK-NEXT:  .LBB3_36: # %cmpxchg.nostore120
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_38
+; CHECK-NEXT:  .LBB3_37: # %cmpxchg.success121
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_38: # %cmpxchg.end118
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 30, 1
+; CHECK-NEXT:    isel 6, 30, 6, 20
+; CHECK-NEXT:    lbz 30, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    lbarx 29, 0, 7
+; CHECK-NEXT:    clrlwi 29, 29, 24
+; CHECK-NEXT:    cmplw 29, 6
+; CHECK-NEXT:    bne 0, .LBB3_42
+; CHECK-NEXT:  # %bb.39: # %cmpxchg.fencedstore105
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_40: # %cmpxchg.trystore104
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stbcx. 30, 0, 7
+; CHECK-NEXT:    beq 0, .LBB3_43
+; CHECK-NEXT:  # %bb.41: # %cmpxchg.releasedload103
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lbarx 29, 0, 7
+; CHECK-NEXT:    clrlwi 29, 29, 24
+; CHECK-NEXT:    cmplw 29, 6
+; CHECK-NEXT:    beq 0, .LBB3_40
+; CHECK-NEXT:  .LBB3_42: # %cmpxchg.nostore101
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_44
+; CHECK-NEXT:  .LBB3_43: # %cmpxchg.success102
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_44: # %cmpxchg.end99
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    lbz 7, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    lharx 30, 0, 8
+; CHECK-NEXT:    clrlwi 30, 30, 16
+; CHECK-NEXT:    cmplw 30, 6
+; CHECK-NEXT:    bne 0, .LBB3_48
+; CHECK-NEXT:  # %bb.45: # %cmpxchg.fencedstore86
+; CHECK-NEXT:    extsb 7, 7
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    clrlwi 7, 7, 16
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_46: # %cmpxchg.trystore85
+; CHECK-NEXT:    #
+; CHECK-NEXT:    sthcx. 7, 0, 8
+; CHECK-NEXT:    beq 0, .LBB3_49
+; CHECK-NEXT:  # %bb.47: # %cmpxchg.releasedload84
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lharx 30, 0, 8
+; CHECK-NEXT:    clrlwi 30, 30, 16
+; CHECK-NEXT:    cmplw 30, 6
+; CHECK-NEXT:    beq 0, .LBB3_46
+; CHECK-NEXT:  .LBB3_48: # %cmpxchg.nostore82
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_50
+; CHECK-NEXT:  .LBB3_49: # %cmpxchg.success83
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_50: # %cmpxchg.end80
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    lbz 7, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    lharx 8, 0, 9
+; CHECK-NEXT:    clrlwi 8, 8, 16
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    bne 0, .LBB3_54
+; CHECK-NEXT:  # %bb.51: # %cmpxchg.fencedstore67
+; CHECK-NEXT:    extsb 7, 7
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    clrlwi 7, 7, 16
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_52: # %cmpxchg.trystore66
+; CHECK-NEXT:    #
+; CHECK-NEXT:    sthcx. 7, 0, 9
+; CHECK-NEXT:    beq 0, .LBB3_55
+; CHECK-NEXT:  # %bb.53: # %cmpxchg.releasedload65
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lharx 8, 0, 9
+; CHECK-NEXT:    clrlwi 8, 8, 16
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    beq 0, .LBB3_52
+; CHECK-NEXT:  .LBB3_54: # %cmpxchg.nostore63
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_56
+; CHECK-NEXT:  .LBB3_55: # %cmpxchg.success64
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_56: # %cmpxchg.end61
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    lbz 7, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    lwarx 8, 0, 10
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    bne 0, .LBB3_60
+; CHECK-NEXT:  # %bb.57: # %cmpxchg.fencedstore48
+; CHECK-NEXT:    extsb 7, 7
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_58: # %cmpxchg.trystore47
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stwcx. 7, 0, 10
+; CHECK-NEXT:    beq 0, .LBB3_61
+; CHECK-NEXT:  # %bb.59: # %cmpxchg.releasedload46
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lwarx 8, 0, 10
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    beq 0, .LBB3_58
+; CHECK-NEXT:  .LBB3_60: # %cmpxchg.nostore44
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_62
+; CHECK-NEXT:  .LBB3_61: # %cmpxchg.success45
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_62: # %cmpxchg.end42
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    lbz 7, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    lwarx 8, 0, 11
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    bne 0, .LBB3_66
+; CHECK-NEXT:  # %bb.63: # %cmpxchg.fencedstore29
+; CHECK-NEXT:    extsb 7, 7
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_64: # %cmpxchg.trystore28
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stwcx. 7, 0, 11
+; CHECK-NEXT:    beq 0, .LBB3_67
+; CHECK-NEXT:  # %bb.65: # %cmpxchg.releasedload27
+; CHECK-NEXT:    #
+; CHECK-NEXT:    lwarx 8, 0, 11
+; CHECK-NEXT:    cmplw 8, 6
+; CHECK-NEXT:    beq 0, .LBB3_64
+; CHECK-NEXT:  .LBB3_66: # %cmpxchg.nostore25
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_68
+; CHECK-NEXT:  .LBB3_67: # %cmpxchg.success26
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_68: # %cmpxchg.end23
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    lbz 7, sc@toc@l(4)
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    lbz 6, uc@toc@l(3)
+; CHECK-NEXT:    ldarx 8, 0, 12
+; CHECK-NEXT:    cmpld 8, 6
+; CHECK-NEXT:    bne 0, .LBB3_72
+; CHECK-NEXT:  # %bb.69: # %cmpxchg.fencedstore10
+; CHECK-NEXT:    extsb 7, 7
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_70: # %cmpxchg.trystore9
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stdcx. 7, 0, 12
+; CHECK-NEXT:    beq 0, .LBB3_73
+; CHECK-NEXT:  # %bb.71: # %cmpxchg.releasedload8
+; CHECK-NEXT:    #
+; CHECK-NEXT:    ldarx 8, 0, 12
+; CHECK-NEXT:    cmpld 8, 6
+; CHECK-NEXT:    beq 0, .LBB3_70
+; CHECK-NEXT:  .LBB3_72: # %cmpxchg.nostore6
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_74
+; CHECK-NEXT:  .LBB3_73: # %cmpxchg.success7
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_74: # %cmpxchg.end4
+; CHECK-NEXT:    li 6, 0
+; CHECK-NEXT:    li 7, 1
+; CHECK-NEXT:    lbz 3, uc@toc@l(3)
+; CHECK-NEXT:    lbz 4, sc@toc@l(4)
+; CHECK-NEXT:    isel 6, 7, 6, 20
+; CHECK-NEXT:    stw 6, ui@toc@l(5)
+; CHECK-NEXT:    ldarx 6, 0, 0
+; CHECK-NEXT:    cmpld 6, 3
+; CHECK-NEXT:    bne 0, .LBB3_78
+; CHECK-NEXT:  # %bb.75: # %cmpxchg.fencedstore
+; CHECK-NEXT:    extsb 4, 4
+; CHECK-NEXT:    sync
+; CHECK-NEXT:    .p2align 5
+; CHECK-NEXT:  .LBB3_76: # %cmpxchg.trystore
+; CHECK-NEXT:    #
+; CHECK-NEXT:    stdcx. 4, 0, 0
+; CHECK-NEXT:    beq 0, .LBB3_79
+; CHECK-NEXT:  # %bb.77: # %cmpxchg.releasedload
+; CHECK-NEXT:    #
+; CHECK-NEXT:    ldarx 6, 0, 0
+; CHECK-NEXT:    cmpld 6, 3
+; CHECK-NEXT:    beq 0, .LBB3_76
+; CHECK-NEXT:  .LBB3_78: # %cmpxchg.nostore
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:    b .LBB3_80
+; CHECK-NEXT:  .LBB3_79: # %cmpxchg.success
+; CHECK-NEXT:    lwsync
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:  .LBB3_80: # %cmpxchg.end
+; CHECK-NEXT:    li 3, 0
+; CHECK-NEXT:    li 4, 1
+; CHECK-NEXT:    ld 30, -16(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 29, -24(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 28, -32(1) # 8-byte Folded Reload
+; CHECK-NEXT:    ld 27, -40(1) # 8-byte Folded Reload
+; CHECK-NEXT:    isel 3, 4, 3, 20
+; CHECK-NEXT:    stw 3, ui@toc@l(5)
+; CHECK-NEXT:    blr
 ;
 ; AIX32-LABEL: test_compare_and_swap:
 ; AIX32:       # %bb.0: # %entry
 ; AIX32-NEXT:    mflr 0
-; AIX32-NEXT:   stwu 1, -144(1)
-; AIX32-NEXT:   stw 0, 152(1)
-; AIX32-NEXT:   stw 29, 132(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   lwz 29, L..C0(2)                        # @sc
-; AIX32-NEXT:   stw 26, 120(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   not     3, 29
-; AIX32-NEXT:   stw 30, 136(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   lwz 30, L..C1(2)                        # @uc
-; AIX32-NEXT:   lbz 4, 0(30)
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   stw 27, 124(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   rlwinm 27, 29, 0, 0, 29
-; AIX32-NEXT:   stw 14, 72(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 15, 76(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   rlwinm 26, 3, 3, 27, 28
-; AIX32-NEXT:   li 3, 255
-; AIX32-NEXT:   slw 3, 3, 26
-; AIX32-NEXT:   stw 16, 80(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 17, 84(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 18, 88(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 19, 92(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 20, 96(1)                           # 4-byte Folded Spill
-; AIX32-NEXT:   stw 21, 100(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 22, 104(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 23, 108(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 24, 112(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 25, 116(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 28, 128(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   stw 31, 140(1)                          # 4-byte Folded Spill
-; AIX32-NEXT:   not     25, 3
-; AIX32-NEXT:   lwarx 3, 0, 27
-; AIX32-NEXT:   srw 6, 3, 26
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 4
-; AIX32-NEXT:   bne     0, L..BB3_4
-; AIX32-NEXT:  # %bb.1:                                # %cmpxchg.fencedstore289
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   slw 5, 5, 26
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_2:                               # %cmpxchg.trystore288
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 6, 3, 25
-; AIX32-NEXT:   or 6, 6, 5
-; AIX32-NEXT:   stwcx. 6, 0, 27
-; AIX32-NEXT:   beq     0, L..BB3_4
-; AIX32-NEXT:  # %bb.3:                                # %cmpxchg.releasedload287
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_2 Depth=1
-; AIX32-NEXT:   lwarx 3, 0, 27
-; AIX32-NEXT:   srw 6, 3, 26
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 4
-; AIX32-NEXT:   beq     0, L..BB3_2
-; AIX32-NEXT:  L..BB3_4:                               # %cmpxchg.nostore285
-; AIX32-NEXT:   not     4, 30
-; AIX32-NEXT:   srw 5, 3, 26
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   rlwinm 24, 30, 0, 0, 29
-; AIX32-NEXT:   rlwinm 23, 4, 3, 27, 28
-; AIX32-NEXT:   li 4, 255
-; AIX32-NEXT:   stb 5, 0(29)
-; AIX32-NEXT:   slw 4, 4, 23
-; AIX32-NEXT:   not     22, 4
-; AIX32-NEXT:   lwarx 4, 0, 24
-; AIX32-NEXT:   srw 6, 4, 23
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   bne     0, L..BB3_8
-; AIX32-NEXT:  # %bb.5:                                # %cmpxchg.fencedstore256
-; AIX32-NEXT:   clrlwi  5, 5, 24
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   slw 5, 5, 23
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_6:                               # %cmpxchg.trystore255
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 6, 4, 22
-; AIX32-NEXT:   or 6, 6, 5
-; AIX32-NEXT:   stwcx. 6, 0, 24
-; AIX32-NEXT:   beq     0, L..BB3_8
-; AIX32-NEXT:  # %bb.7:                                # %cmpxchg.releasedload254
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_6 Depth=1
-; AIX32-NEXT:   lwarx 4, 0, 24
-; AIX32-NEXT:   srw 6, 4, 23
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   beq     0, L..BB3_6
-; AIX32-NEXT:  L..BB3_8:                               # %cmpxchg.nostore252
-; AIX32-NEXT:   srw 4, 4, 23
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   lis 3, 0
-; AIX32-NEXT:   lbz 7, 0(29)
-; AIX32-NEXT:   stb 4, 0(30)
-; AIX32-NEXT:   clrlwi  6, 4, 24
-; AIX32-NEXT:   lwz 4, L..C2(2)                         # @ss
-; AIX32-NEXT:   ori 3, 3, 65535
-; AIX32-NEXT:   clrlwi  5, 4, 30
-; AIX32-NEXT:   rlwinm 21, 4, 0, 0, 29
-; AIX32-NEXT:   xori 5, 5, 2
-; AIX32-NEXT:   slwi 20, 5, 3
-; AIX32-NEXT:   slw 5, 3, 20
-; AIX32-NEXT:   not     19, 5
-; AIX32-NEXT:   lwarx 5, 0, 21
-; AIX32-NEXT:   srw 8, 5, 20
-; AIX32-NEXT:   clrlwi  8, 8, 16
-; AIX32-NEXT:   cmplw   8, 6
-; AIX32-NEXT:   bne     0, L..BB3_12
-; AIX32-NEXT:  # %bb.9:                                # %cmpxchg.fencedstore223
-; AIX32-NEXT:   extsb 7, 7
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   clrlwi  7, 7, 16
-; AIX32-NEXT:   slw 7, 7, 20
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_10:                              # %cmpxchg.trystore222
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 8, 5, 19
-; AIX32-NEXT:   or 8, 8, 7
-; AIX32-NEXT:   stwcx. 8, 0, 21
-; AIX32-NEXT:   beq     0, L..BB3_12
-; AIX32-NEXT:  # %bb.11:                               # %cmpxchg.releasedload221
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_10 Depth=1
-; AIX32-NEXT:   lwarx 5, 0, 21
-; AIX32-NEXT:   srw 8, 5, 20
-; AIX32-NEXT:   clrlwi  8, 8, 16
-; AIX32-NEXT:   cmplw   8, 6
-; AIX32-NEXT:   beq     0, L..BB3_10
-; AIX32-NEXT:  L..BB3_12:                              # %cmpxchg.nostore219
-; AIX32-NEXT:   srw 5, 5, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   lbz 6, 0(29)
-; AIX32-NEXT:   sth 5, 0(4)
-; AIX32-NEXT:   lwz 4, L..C3(2)                         # @us
-; AIX32-NEXT:   lbz 5, 0(30)
-; AIX32-NEXT:   clrlwi  7, 4, 30
-; AIX32-NEXT:   rlwinm 18, 4, 0, 0, 29
-; AIX32-NEXT:   xori 7, 7, 2
-; AIX32-NEXT:   slwi 17, 7, 3
-; AIX32-NEXT:   slw 3, 3, 17
-; AIX32-NEXT:   not     16, 3
-; AIX32-NEXT:   lwarx 3, 0, 18
-; AIX32-NEXT:   srw 7, 3, 17
-; AIX32-NEXT:   clrlwi  7, 7, 16
-; AIX32-NEXT:   cmplw   7, 5
-; AIX32-NEXT:   bne     0, L..BB3_16
-; AIX32-NEXT:  # %bb.13:                               # %cmpxchg.fencedstore190
-; AIX32-NEXT:   extsb 6, 6
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   clrlwi  6, 6, 16
-; AIX32-NEXT:   slw 6, 6, 17
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_14:                              # %cmpxchg.trystore189
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 7, 3, 16
-; AIX32-NEXT:   or 7, 7, 6
-; AIX32-NEXT:   stwcx. 7, 0, 18
-; AIX32-NEXT:   beq     0, L..BB3_16
-; AIX32-NEXT:  # %bb.15:                               # %cmpxchg.releasedload188
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_14 Depth=1
-; AIX32-NEXT:   lwarx 3, 0, 18
-; AIX32-NEXT:   srw 7, 3, 17
-; AIX32-NEXT:   clrlwi  7, 7, 16
-; AIX32-NEXT:   cmplw   7, 5
-; AIX32-NEXT:   beq     0, L..BB3_14
-; AIX32-NEXT:  L..BB3_16:                              # %cmpxchg.nostore186
-; AIX32-NEXT:   srw 3, 3, 17
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   lwz 15, L..C4(2)                        # @si
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   sth 3, 0(4)
-; AIX32-NEXT:   lbz 4, 0(30)
-; AIX32-NEXT:   lwarx 3, 0, 15
-; AIX32-NEXT:   cmplw   3, 4
-; AIX32-NEXT:   bne     0, L..BB3_20
-; AIX32-NEXT:  # %bb.17:                               # %cmpxchg.fencedstore171
-; AIX32-NEXT:   extsb 5, 5
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   .align  5
-; AIX32-NEXT:  L..BB3_18:                              # %cmpxchg.trystore170
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   stwcx. 5, 0, 15
-; AIX32-NEXT:   beq     0, L..BB3_20
-; AIX32-NEXT:  # %bb.19:                               # %cmpxchg.releasedload169
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_18 Depth=1
-; AIX32-NEXT:   lwarx 3, 0, 15
-; AIX32-NEXT:   cmplw   3, 4
-; AIX32-NEXT:   beq     0, L..BB3_18
-; AIX32-NEXT:  L..BB3_20:                              # %cmpxchg.nostore167
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   lwz 28, L..C5(2)                        # @ui
-; AIX32-NEXT:   stw 3, 0(15)
-; AIX32-NEXT:   lbz 4, 0(30)
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   lwarx 3, 0, 28
-; AIX32-NEXT:   cmplw   3, 4
-; AIX32-NEXT:   bne     0, L..BB3_24
-; AIX32-NEXT:  # %bb.21:                               # %cmpxchg.fencedstore152
-; AIX32-NEXT:   extsb 5, 5
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   .align  5
-; AIX32-NEXT:  L..BB3_22:                              # %cmpxchg.trystore151
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   stwcx. 5, 0, 28
-; AIX32-NEXT:   beq     0, L..BB3_24
-; AIX32-NEXT:  # %bb.23:                               # %cmpxchg.releasedload150
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_22 Depth=1
-; AIX32-NEXT:   lwarx 3, 0, 28
-; AIX32-NEXT:   cmplw   3, 4
-; AIX32-NEXT:   beq     0, L..BB3_22
-; AIX32-NEXT:  L..BB3_24:                              # %cmpxchg.nostore148
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lwz 31, L..C6(2)                        # @sll
-; AIX32-NEXT:   lbz 3, 0(29)
-; AIX32-NEXT:   li 14, 0
-; AIX32-NEXT:   addi 4, 1, 64
-; AIX32-NEXT:   li 7, 5
-; AIX32-NEXT:   li 8, 5
-; AIX32-NEXT:   stw 14, 64(1)
-; AIX32-NEXT:   extsb 6, 3
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   srawi 5, 6, 31
-; AIX32-NEXT:   stw 3, 68(1)
-; AIX32-NEXT:   mr      3, 31
-; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:   nop
-; AIX32-NEXT:   lwz 3, 68(1)
-; AIX32-NEXT:   lbz 4, 0(29)
-; AIX32-NEXT:   li 7, 5
-; AIX32-NEXT:   li 8, 5
-; AIX32-NEXT:   stw 3, 4(31)
-; AIX32-NEXT:   lwz 3, 64(1)
-; AIX32-NEXT:   extsb 6, 4
-; AIX32-NEXT:   addi 4, 1, 64
-; AIX32-NEXT:   stw 14, 64(1)
-; AIX32-NEXT:   srawi 5, 6, 31
-; AIX32-NEXT:   stw 3, 0(31)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwz 31, L..C7(2)                        # @ull
-; AIX32-NEXT:   stw 3, 68(1)
-; AIX32-NEXT:   mr      3, 31
-; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:   nop
-; AIX32-NEXT:   lwz 3, 64(1)
-; AIX32-NEXT:   lwz 4, 68(1)
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   stw 4, 4(31)
-; AIX32-NEXT:   stw 3, 0(31)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 4, 0, 27
-; AIX32-NEXT:   srw 6, 4, 26
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   bne     0, L..BB3_28
-; AIX32-NEXT:  # %bb.25:                               # %cmpxchg.fencedstore119
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   slw 5, 5, 26
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_26:                              # %cmpxchg.trystore118
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 4, 4, 25
-; AIX32-NEXT:   or 4, 4, 5
-; AIX32-NEXT:   stwcx. 4, 0, 27
-; AIX32-NEXT:   beq     0, L..BB3_29
-; AIX32-NEXT:  # %bb.27:                               # %cmpxchg.releasedload117
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_26 Depth=1
-; AIX32-NEXT:   lwarx 4, 0, 27
-; AIX32-NEXT:   srw 6, 4, 26
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   beq     0, L..BB3_26
-; AIX32-NEXT:  L..BB3_28:                              # %cmpxchg.nostore115
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_30
-; AIX32-NEXT:  L..BB3_29:                              # %cmpxchg.success116
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_30:                              # %cmpxchg.end113
-; AIX32-NEXT:   li 3, 0
-; AIX32-NEXT:   li 4, 1
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   isel 3, 4, 3, 20
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 4, 0, 24
-; AIX32-NEXT:   srw 6, 4, 23
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   bne     0, L..BB3_34
-; AIX32-NEXT:  # %bb.31:                               # %cmpxchg.fencedstore86
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   slw 5, 5, 23
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_32:                              # %cmpxchg.trystore85
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 4, 4, 22
-; AIX32-NEXT:   or 4, 4, 5
-; AIX32-NEXT:   stwcx. 4, 0, 24
-; AIX32-NEXT:   beq     0, L..BB3_35
-; AIX32-NEXT:  # %bb.33:                               # %cmpxchg.releasedload84
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_32 Depth=1
-; AIX32-NEXT:   lwarx 4, 0, 24
-; AIX32-NEXT:   srw 6, 4, 23
-; AIX32-NEXT:   clrlwi  6, 6, 24
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   beq     0, L..BB3_32
-; AIX32-NEXT:  L..BB3_34:                              # %cmpxchg.nostore82
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_36
-; AIX32-NEXT:  L..BB3_35:                              # %cmpxchg.success83
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_36:                              # %cmpxchg.end80
-; AIX32-NEXT:   li 3, 0
-; AIX32-NEXT:   li 4, 1
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   isel 3, 4, 3, 20
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 4, 0, 21
-; AIX32-NEXT:   srw 6, 4, 20
-; AIX32-NEXT:   clrlwi  6, 6, 16
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   bne     0, L..BB3_40
-; AIX32-NEXT:  # %bb.37:                               # %cmpxchg.fencedstore53
-; AIX32-NEXT:   extsb 5, 5
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   clrlwi  5, 5, 16
-; AIX32-NEXT:   slw 5, 5, 20
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_38:                              # %cmpxchg.trystore52
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 4, 4, 19
-; AIX32-NEXT:   or 4, 4, 5
-; AIX32-NEXT:   stwcx. 4, 0, 21
-; AIX32-NEXT:   beq     0, L..BB3_41
-; AIX32-NEXT:  # %bb.39:                               # %cmpxchg.releasedload51
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_38 Depth=1
-; AIX32-NEXT:   lwarx 4, 0, 21
-; AIX32-NEXT:   srw 6, 4, 20
-; AIX32-NEXT:   clrlwi  6, 6, 16
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   beq     0, L..BB3_38
-; AIX32-NEXT:  L..BB3_40:                              # %cmpxchg.nostore49
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_42
-; AIX32-NEXT:  L..BB3_41:                              # %cmpxchg.success50
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_42:                              # %cmpxchg.end47
-; AIX32-NEXT:   li 3, 0
-; AIX32-NEXT:   li 4, 1
-; AIX32-NEXT:   lbz 5, 0(29)
-; AIX32-NEXT:   isel 3, 4, 3, 20
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 4, 0, 18
-; AIX32-NEXT:   srw 6, 4, 17
-; AIX32-NEXT:   clrlwi  6, 6, 16
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   bne     0, L..BB3_46
-; AIX32-NEXT:  # %bb.43:                               # %cmpxchg.fencedstore29
-; AIX32-NEXT:   extsb 5, 5
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   clrlwi  5, 5, 16
-; AIX32-NEXT:   slw 5, 5, 17
-; AIX32-NEXT:   .align  4
-; AIX32-NEXT:  L..BB3_44:                              # %cmpxchg.trystore28
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   and 4, 4, 16
-; AIX32-NEXT:   or 4, 4, 5
-; AIX32-NEXT:   stwcx. 4, 0, 18
-; AIX32-NEXT:   beq     0, L..BB3_47
-; AIX32-NEXT:  # %bb.45:                               # %cmpxchg.releasedload27
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_44 Depth=1
-; AIX32-NEXT:   lwarx 4, 0, 18
-; AIX32-NEXT:   srw 6, 4, 17
-; AIX32-NEXT:   clrlwi  6, 6, 16
-; AIX32-NEXT:   cmplw   6, 3
-; AIX32-NEXT:   beq     0, L..BB3_44
-; AIX32-NEXT:  L..BB3_46:                              # %cmpxchg.nostore25
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_48
-; AIX32-NEXT:  L..BB3_47:                              # %cmpxchg.success26
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_48:                              # %cmpxchg.end23
-; AIX32-NEXT:   li 3, 0
-; AIX32-NEXT:   li 4, 1
-; AIX32-NEXT:   isel 3, 4, 3, 20
-; AIX32-NEXT:   lbz 4, 0(29)
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 5, 0, 15
-; AIX32-NEXT:   cmplw   5, 3
-; AIX32-NEXT:   bne     0, L..BB3_52
-; AIX32-NEXT:  # %bb.49:                               # %cmpxchg.fencedstore10
-; AIX32-NEXT:   extsb 4, 4
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   .align  5
-; AIX32-NEXT:  L..BB3_50:                              # %cmpxchg.trystore9
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   stwcx. 4, 0, 15
-; AIX32-NEXT:   beq     0, L..BB3_53
-; AIX32-NEXT:  # %bb.51:                               # %cmpxchg.releasedload8
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_50 Depth=1
-; AIX32-NEXT:   lwarx 5, 0, 15
-; AIX32-NEXT:   cmplw   5, 3
-; AIX32-NEXT:   beq     0, L..BB3_50
-; AIX32-NEXT:  L..BB3_52:                              # %cmpxchg.nostore6
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_54
-; AIX32-NEXT:  L..BB3_53:                              # %cmpxchg.success7
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_54:                              # %cmpxchg.end4
-; AIX32-NEXT:   li 3, 0
-; AIX32-NEXT:   li 4, 1
-; AIX32-NEXT:   isel 3, 4, 3, 20
-; AIX32-NEXT:   lbz 4, 0(29)
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   lwarx 5, 0, 28
-; AIX32-NEXT:   cmplw   5, 3
-; AIX32-NEXT:   bne     0, L..BB3_58
-; AIX32-NEXT:  # %bb.55:                               # %cmpxchg.fencedstore
-; AIX32-NEXT:   extsb 4, 4
-; AIX32-NEXT:   sync
-; AIX32-NEXT:   .align  5
-; AIX32-NEXT:  L..BB3_56:                              # %cmpxchg.trystore
-; AIX32-NEXT:                                          # =>This Inner Loop Header: Depth=1
-; AIX32-NEXT:   stwcx. 4, 0, 28
-; AIX32-NEXT:   beq     0, L..BB3_59
-; AIX32-NEXT:  # %bb.57:                               # %cmpxchg.releasedload
-; AIX32-NEXT:                                          #   in Loop: Header=BB3_56 Depth=1
-; AIX32-NEXT:   lwarx 5, 0, 28
-; AIX32-NEXT:   cmplw   5, 3
-; AIX32-NEXT:   beq     0, L..BB3_56
-; AIX32-NEXT:  L..BB3_58:                              # %cmpxchg.nostore
-; AIX32-NEXT:   crxor 20, 20, 20
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   b L..BB3_60
-; AIX32-NEXT:  L..BB3_59:                              # %cmpxchg.success
-; AIX32-NEXT:   lwsync
-; AIX32-NEXT:   creqv 20, 20, 20
-; AIX32-NEXT:  L..BB3_60:                              # %cmpxchg.end
-; AIX32-NEXT:   li 3, 1
-; AIX32-NEXT:   li 31, 0
-; AIX32-NEXT:   lbz 4, 0(29)
-; AIX32-NEXT:   isel 3, 3, 31, 20
-; AIX32-NEXT:   li 7, 5
-; AIX32-NEXT:   li 8, 5
-; AIX32-NEXT:   extsb 6, 4
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   addi 4, 1, 64
-; AIX32-NEXT:   stw 31, 64(1)
-; AIX32-NEXT:   srawi 5, 6, 31
-; AIX32-NEXT:   stw 3, 68(1)
-; AIX32-NEXT:   lwz 3, L..C6(2)                         # @sll
-; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:   nop
-; AIX32-NEXT:   lbz 4, 0(29)
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lbz 3, 0(30)
-; AIX32-NEXT:   li 7, 5
-; AIX32-NEXT:   li 8, 5
-; AIX32-NEXT:   extsb 6, 4
-; AIX32-NEXT:   stw 3, 68(1)
-; AIX32-NEXT:   lwz 3, L..C7(2)                         # @ull
-; AIX32-NEXT:   addi 4, 1, 64
-; AIX32-NEXT:   stw 31, 64(1)
-; AIX32-NEXT:   srawi 5, 6, 31
-; AIX32-NEXT:   bl .__atomic_compare_exchange_8[PR]
-; AIX32-NEXT:   nop
-; AIX32-NEXT:   stw 3, 0(28)
-; AIX32-NEXT:   lwz 31, 140(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 30, 136(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 29, 132(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 28, 128(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 27, 124(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 26, 120(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 25, 116(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 24, 112(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 23, 108(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 22, 104(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 21, 100(1)                          # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 20, 96(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 19, 92(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 18, 88(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 17, 84(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 16, 80(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 15, 76(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   lwz 14, 72(1)                           # 4-byte Folded Reload
-; AIX32-NEXT:   addi 1, 1, 144
-; AIX32-NEXT:   lwz 0, 8(1)
-; AIX32-NEXT:   mtlr 0
+; AIX32-NEXT:    stwu 1, -144(1)
+; AIX32-NEXT:    stw 0, 152(1)
+; AIX32-NEXT:    stw 29, 132(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 29, L..C0(2) # @sc
+; AIX32-NEXT:    stw 26, 120(1) # 4-byte Folded Spill
+; AIX32-NEXT:    not 3, 29
+; AIX32-NEXT:    stw 30, 136(1) # 4-byte Folded Spill
+; AIX32-NEXT:    lwz 30, L..C1(2) # @uc
+; AIX32-NEXT:    lbz 4, 0(30)
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    stw 27, 124(1) # 4-byte Folded Spill
+; AIX32-NEXT:    rlwinm 27, 29, 0, 0, 29
+; AIX32-NEXT:    stw 14, 72(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 15, 76(1) # 4-byte Folded Spill
+; AIX32-NEXT:    rlwinm 26, 3, 3, 27, 28
+; AIX32-NEXT:    li 3, 255
+; AIX32-NEXT:    slw 3, 3, 26
+; AIX32-NEXT:    stw 16, 80(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 17, 84(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 18, 88(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 19, 92(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 20, 96(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 21, 100(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 22, 104(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 23, 108(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 24, 112(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 25, 116(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 28, 128(1) # 4-byte Folded Spill
+; AIX32-NEXT:    stw 31, 140(1) # 4-byte Folded Spill
+; AIX32-NEXT:    not 25, 3
+; AIX32-NEXT:    lwarx 3, 0, 27
+; AIX32-NEXT:    srw 6, 3, 26
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 4
+; AIX32-NEXT:    bne 0, L..BB3_4
+; AIX32-NEXT:  # %bb.1: # %cmpxchg.fencedstore289
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    slw 5, 5, 26
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_2: # %cmpxchg.trystore288
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 6, 3, 25
+; AIX32-NEXT:    or 6, 6, 5
+; AIX32-NEXT:    stwcx. 6, 0, 27
+; AIX32-NEXT:    beq 0, L..BB3_4
+; AIX32-NEXT:  # %bb.3: # %cmpxchg.releasedload287
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 3, 0, 27
+; AIX32-NEXT:    srw 6, 3, 26
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 4
+; AIX32-NEXT:    beq 0, L..BB3_2
+; AIX32-NEXT:  L..BB3_4: # %cmpxchg.nostore285
+; AIX32-NEXT:    not 4, 30
+; AIX32-NEXT:    srw 5, 3, 26
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    rlwinm 24, 30, 0, 0, 29
+; AIX32-NEXT:    rlwinm 23, 4, 3, 27, 28
+; AIX32-NEXT:    li 4, 255
+; AIX32-NEXT:    stb 5, 0(29)
+; AIX32-NEXT:    slw 4, 4, 23
+; AIX32-NEXT:    not 22, 4
+; AIX32-NEXT:    lwarx 4, 0, 24
+; AIX32-NEXT:    srw 6, 4, 23
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    bne 0, L..BB3_8
+; AIX32-NEXT:  # %bb.5: # %cmpxchg.fencedstore256
+; AIX32-NEXT:    clrlwi 5, 5, 24
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    slw 5, 5, 23
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_6: # %cmpxchg.trystore255
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 6, 4, 22
+; AIX32-NEXT:    or 6, 6, 5
+; AIX32-NEXT:    stwcx. 6, 0, 24
+; AIX32-NEXT:    beq 0, L..BB3_8
+; AIX32-NEXT:  # %bb.7: # %cmpxchg.releasedload254
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 4, 0, 24
+; AIX32-NEXT:    srw 6, 4, 23
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    beq 0, L..BB3_6
+; AIX32-NEXT:  L..BB3_8: # %cmpxchg.nostore252
+; AIX32-NEXT:    srw 4, 4, 23
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    lis 3, 0
+; AIX32-NEXT:    lbz 7, 0(29)
+; AIX32-NEXT:    stb 4, 0(30)
+; AIX32-NEXT:    clrlwi 6, 4, 24
+; AIX32-NEXT:    lwz 4, L..C2(2) # @ss
+; AIX32-NEXT:    ori 3, 3, 65535
+; AIX32-NEXT:    clrlwi 5, 4, 30
+; AIX32-NEXT:    rlwinm 21, 4, 0, 0, 29
+; AIX32-NEXT:    xori 5, 5, 2
+; AIX32-NEXT:    slwi 20, 5, 3
+; AIX32-NEXT:    slw 5, 3, 20
+; AIX32-NEXT:    not 19, 5
+; AIX32-NEXT:    lwarx 5, 0, 21
+; AIX32-NEXT:    srw 8, 5, 20
+; AIX32-NEXT:    clrlwi 8, 8, 16
+; AIX32-NEXT:    cmplw 8, 6
+; AIX32-NEXT:    bne 0, L..BB3_12
+; AIX32-NEXT:  # %bb.9: # %cmpxchg.fencedstore223
+; AIX32-NEXT:    extsb 7, 7
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    clrlwi 7, 7, 16
+; AIX32-NEXT:    slw 7, 7, 20
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_10: # %cmpxchg.trystore222
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 8, 5, 19
+; AIX32-NEXT:    or 8, 8, 7
+; AIX32-NEXT:    stwcx. 8, 0, 21
+; AIX32-NEXT:    beq 0, L..BB3_12
+; AIX32-NEXT:  # %bb.11: # %cmpxchg.releasedload221
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 5, 0, 21
+; AIX32-NEXT:    srw 8, 5, 20
+; AIX32-NEXT:    clrlwi 8, 8, 16
+; AIX32-NEXT:    cmplw 8, 6
+; AIX32-NEXT:    beq 0, L..BB3_10
+; AIX32-NEXT:  L..BB3_12: # %cmpxchg.nostore219
+; AIX32-NEXT:    srw 5, 5, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    lbz 6, 0(29)
+; AIX32-NEXT:    sth 5, 0(4)
+; AIX32-NEXT:    lwz 4, L..C3(2) # @us
+; AIX32-NEXT:    lbz 5, 0(30)
+; AIX32-NEXT:    clrlwi 7, 4, 30
+; AIX32-NEXT:    rlwinm 18, 4, 0, 0, 29
+; AIX32-NEXT:    xori 7, 7, 2
+; AIX32-NEXT:    slwi 17, 7, 3
+; AIX32-NEXT:    slw 3, 3, 17
+; AIX32-NEXT:    not 16, 3
+; AIX32-NEXT:    lwarx 3, 0, 18
+; AIX32-NEXT:    srw 7, 3, 17
+; AIX32-NEXT:    clrlwi 7, 7, 16
+; AIX32-NEXT:    cmplw 7, 5
+; AIX32-NEXT:    bne 0, L..BB3_16
+; AIX32-NEXT:  # %bb.13: # %cmpxchg.fencedstore190
+; AIX32-NEXT:    extsb 6, 6
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    clrlwi 6, 6, 16
+; AIX32-NEXT:    slw 6, 6, 17
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_14: # %cmpxchg.trystore189
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 7, 3, 16
+; AIX32-NEXT:    or 7, 7, 6
+; AIX32-NEXT:    stwcx. 7, 0, 18
+; AIX32-NEXT:    beq 0, L..BB3_16
+; AIX32-NEXT:  # %bb.15: # %cmpxchg.releasedload188
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 3, 0, 18
+; AIX32-NEXT:    srw 7, 3, 17
+; AIX32-NEXT:    clrlwi 7, 7, 16
+; AIX32-NEXT:    cmplw 7, 5
+; AIX32-NEXT:    beq 0, L..BB3_14
+; AIX32-NEXT:  L..BB3_16: # %cmpxchg.nostore186
+; AIX32-NEXT:    srw 3, 3, 17
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    lwz 15, L..C4(2) # @si
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    sth 3, 0(4)
+; AIX32-NEXT:    lbz 4, 0(30)
+; AIX32-NEXT:    lwarx 3, 0, 15
+; AIX32-NEXT:    cmplw 3, 4
+; AIX32-NEXT:    bne 0, L..BB3_20
+; AIX32-NEXT:  # %bb.17: # %cmpxchg.fencedstore171
+; AIX32-NEXT:    extsb 5, 5
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    .align 5
+; AIX32-NEXT:  L..BB3_18: # %cmpxchg.trystore170
+; AIX32-NEXT:    #
+; AIX32-NEXT:    stwcx. 5, 0, 15
+; AIX32-NEXT:    beq 0, L..BB3_20
+; AIX32-NEXT:  # %bb.19: # %cmpxchg.releasedload169
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 3, 0, 15
+; AIX32-NEXT:    cmplw 3, 4
+; AIX32-NEXT:    beq 0, L..BB3_18
+; AIX32-NEXT:  L..BB3_20: # %cmpxchg.nostore167
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    lwz 28, L..C5(2) # @ui
+; AIX32-NEXT:    stw 3, 0(15)
+; AIX32-NEXT:    lbz 4, 0(30)
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    lwarx 3, 0, 28
+; AIX32-NEXT:    cmplw 3, 4
+; AIX32-NEXT:    bne 0, L..BB3_24
+; AIX32-NEXT:  # %bb.21: # %cmpxchg.fencedstore152
+; AIX32-NEXT:    extsb 5, 5
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    .align 5
+; AIX32-NEXT:  L..BB3_22: # %cmpxchg.trystore151
+; AIX32-NEXT:    #
+; AIX32-NEXT:    stwcx. 5, 0, 28
+; AIX32-NEXT:    beq 0, L..BB3_24
+; AIX32-NEXT:  # %bb.23: # %cmpxchg.releasedload150
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 3, 0, 28
+; AIX32-NEXT:    cmplw 3, 4
+; AIX32-NEXT:    beq 0, L..BB3_22
+; AIX32-NEXT:  L..BB3_24: # %cmpxchg.nostore148
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lwz 31, L..C6(2) # @sll
+; AIX32-NEXT:    lbz 3, 0(29)
+; AIX32-NEXT:    li 14, 0
+; AIX32-NEXT:    addi 4, 1, 64
+; AIX32-NEXT:    li 7, 5
+; AIX32-NEXT:    li 8, 5
+; AIX32-NEXT:    stw 14, 64(1)
+; AIX32-NEXT:    extsb 6, 3
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    srawi 5, 6, 31
+; AIX32-NEXT:    stw 3, 68(1)
+; AIX32-NEXT:    mr 3, 31
+; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 68(1)
+; AIX32-NEXT:    lbz 4, 0(29)
+; AIX32-NEXT:    li 7, 5
+; AIX32-NEXT:    li 8, 5
+; AIX32-NEXT:    stw 3, 4(31)
+; AIX32-NEXT:    lwz 3, 64(1)
+; AIX32-NEXT:    extsb 6, 4
+; AIX32-NEXT:    addi 4, 1, 64
+; AIX32-NEXT:    stw 14, 64(1)
+; AIX32-NEXT:    srawi 5, 6, 31
+; AIX32-NEXT:    stw 3, 0(31)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwz 31, L..C7(2) # @ull
+; AIX32-NEXT:    stw 3, 68(1)
+; AIX32-NEXT:    mr 3, 31
+; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lwz 3, 64(1)
+; AIX32-NEXT:    lwz 4, 68(1)
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    stw 4, 4(31)
+; AIX32-NEXT:    stw 3, 0(31)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 4, 0, 27
+; AIX32-NEXT:    srw 6, 4, 26
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    bne 0, L..BB3_28
+; AIX32-NEXT:  # %bb.25: # %cmpxchg.fencedstore119
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    slw 5, 5, 26
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_26: # %cmpxchg.trystore118
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 4, 4, 25
+; AIX32-NEXT:    or 4, 4, 5
+; AIX32-NEXT:    stwcx. 4, 0, 27
+; AIX32-NEXT:    beq 0, L..BB3_29
+; AIX32-NEXT:  # %bb.27: # %cmpxchg.releasedload117
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 4, 0, 27
+; AIX32-NEXT:    srw 6, 4, 26
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    beq 0, L..BB3_26
+; AIX32-NEXT:  L..BB3_28: # %cmpxchg.nostore115
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_30
+; AIX32-NEXT:  L..BB3_29: # %cmpxchg.success116
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_30: # %cmpxchg.end113
+; AIX32-NEXT:    li 3, 0
+; AIX32-NEXT:    li 4, 1
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    isel 3, 4, 3, 20
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 4, 0, 24
+; AIX32-NEXT:    srw 6, 4, 23
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    bne 0, L..BB3_34
+; AIX32-NEXT:  # %bb.31: # %cmpxchg.fencedstore86
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    slw 5, 5, 23
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_32: # %cmpxchg.trystore85
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 4, 4, 22
+; AIX32-NEXT:    or 4, 4, 5
+; AIX32-NEXT:    stwcx. 4, 0, 24
+; AIX32-NEXT:    beq 0, L..BB3_35
+; AIX32-NEXT:  # %bb.33: # %cmpxchg.releasedload84
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 4, 0, 24
+; AIX32-NEXT:    srw 6, 4, 23
+; AIX32-NEXT:    clrlwi 6, 6, 24
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    beq 0, L..BB3_32
+; AIX32-NEXT:  L..BB3_34: # %cmpxchg.nostore82
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_36
+; AIX32-NEXT:  L..BB3_35: # %cmpxchg.success83
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_36: # %cmpxchg.end80
+; AIX32-NEXT:    li 3, 0
+; AIX32-NEXT:    li 4, 1
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    isel 3, 4, 3, 20
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 4, 0, 21
+; AIX32-NEXT:    srw 6, 4, 20
+; AIX32-NEXT:    clrlwi 6, 6, 16
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    bne 0, L..BB3_40
+; AIX32-NEXT:  # %bb.37: # %cmpxchg.fencedstore53
+; AIX32-NEXT:    extsb 5, 5
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    clrlwi 5, 5, 16
+; AIX32-NEXT:    slw 5, 5, 20
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_38: # %cmpxchg.trystore52
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 4, 4, 19
+; AIX32-NEXT:    or 4, 4, 5
+; AIX32-NEXT:    stwcx. 4, 0, 21
+; AIX32-NEXT:    beq 0, L..BB3_41
+; AIX32-NEXT:  # %bb.39: # %cmpxchg.releasedload51
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 4, 0, 21
+; AIX32-NEXT:    srw 6, 4, 20
+; AIX32-NEXT:    clrlwi 6, 6, 16
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    beq 0, L..BB3_38
+; AIX32-NEXT:  L..BB3_40: # %cmpxchg.nostore49
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_42
+; AIX32-NEXT:  L..BB3_41: # %cmpxchg.success50
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_42: # %cmpxchg.end47
+; AIX32-NEXT:    li 3, 0
+; AIX32-NEXT:    li 4, 1
+; AIX32-NEXT:    lbz 5, 0(29)
+; AIX32-NEXT:    isel 3, 4, 3, 20
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 4, 0, 18
+; AIX32-NEXT:    srw 6, 4, 17
+; AIX32-NEXT:    clrlwi 6, 6, 16
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    bne 0, L..BB3_46
+; AIX32-NEXT:  # %bb.43: # %cmpxchg.fencedstore29
+; AIX32-NEXT:    extsb 5, 5
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    clrlwi 5, 5, 16
+; AIX32-NEXT:    slw 5, 5, 17
+; AIX32-NEXT:    .align 4
+; AIX32-NEXT:  L..BB3_44: # %cmpxchg.trystore28
+; AIX32-NEXT:    #
+; AIX32-NEXT:    and 4, 4, 16
+; AIX32-NEXT:    or 4, 4, 5
+; AIX32-NEXT:    stwcx. 4, 0, 18
+; AIX32-NEXT:    beq 0, L..BB3_47
+; AIX32-NEXT:  # %bb.45: # %cmpxchg.releasedload27
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 4, 0, 18
+; AIX32-NEXT:    srw 6, 4, 17
+; AIX32-NEXT:    clrlwi 6, 6, 16
+; AIX32-NEXT:    cmplw 6, 3
+; AIX32-NEXT:    beq 0, L..BB3_44
+; AIX32-NEXT:  L..BB3_46: # %cmpxchg.nostore25
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_48
+; AIX32-NEXT:  L..BB3_47: # %cmpxchg.success26
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_48: # %cmpxchg.end23
+; AIX32-NEXT:    li 3, 0
+; AIX32-NEXT:    li 4, 1
+; AIX32-NEXT:    isel 3, 4, 3, 20
+; AIX32-NEXT:    lbz 4, 0(29)
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 5, 0, 15
+; AIX32-NEXT:    cmplw 5, 3
+; AIX32-NEXT:    bne 0, L..BB3_52
+; AIX32-NEXT:  # %bb.49: # %cmpxchg.fencedstore10
+; AIX32-NEXT:    extsb 4, 4
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    .align 5
+; AIX32-NEXT:  L..BB3_50: # %cmpxchg.trystore9
+; AIX32-NEXT:    #
+; AIX32-NEXT:    stwcx. 4, 0, 15
+; AIX32-NEXT:    beq 0, L..BB3_53
+; AIX32-NEXT:  # %bb.51: # %cmpxchg.releasedload8
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 5, 0, 15
+; AIX32-NEXT:    cmplw 5, 3
+; AIX32-NEXT:    beq 0, L..BB3_50
+; AIX32-NEXT:  L..BB3_52: # %cmpxchg.nostore6
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_54
+; AIX32-NEXT:  L..BB3_53: # %cmpxchg.success7
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_54: # %cmpxchg.end4
+; AIX32-NEXT:    li 3, 0
+; AIX32-NEXT:    li 4, 1
+; AIX32-NEXT:    isel 3, 4, 3, 20
+; AIX32-NEXT:    lbz 4, 0(29)
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    lwarx 5, 0, 28
+; AIX32-NEXT:    cmplw 5, 3
+; AIX32-NEXT:    bne 0, L..BB3_58
+; AIX32-NEXT:  # %bb.55: # %cmpxchg.fencedstore
+; AIX32-NEXT:    extsb 4, 4
+; AIX32-NEXT:    sync
+; AIX32-NEXT:    .align 5
+; AIX32-NEXT:  L..BB3_56: # %cmpxchg.trystore
+; AIX32-NEXT:    #
+; AIX32-NEXT:    stwcx. 4, 0, 28
+; AIX32-NEXT:    beq 0, L..BB3_59
+; AIX32-NEXT:  # %bb.57: # %cmpxchg.releasedload
+; AIX32-NEXT:    #
+; AIX32-NEXT:    lwarx 5, 0, 28
+; AIX32-NEXT:    cmplw 5, 3
+; AIX32-NEXT:    beq 0, L..BB3_56
+; AIX32-NEXT:  L..BB3_58: # %cmpxchg.nostore
+; AIX32-NEXT:    crxor 20, 20, 20
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    b L..BB3_60
+; AIX32-NEXT:  L..BB3_59: # %cmpxchg.success
+; AIX32-NEXT:    lwsync
+; AIX32-NEXT:    creqv 20, 20, 20
+; AIX32-NEXT:  L..BB3_60: # %cmpxchg.end
+; AIX32-NEXT:    li 3, 1
+; AIX32-NEXT:    li 31, 0
+; AIX32-NEXT:    lbz 4, 0(29)
+; AIX32-NEXT:    isel 3, 3, 31, 20
+; AIX32-NEXT:    li 7, 5
+; AIX32-NEXT:    li 8, 5
+; AIX32-NEXT:    extsb 6, 4
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    addi 4, 1, 64
+; AIX32-NEXT:    stw 31, 64(1)
+; AIX32-NEXT:    srawi 5, 6, 31
+; AIX32-NEXT:    stw 3, 68(1)
+; AIX32-NEXT:    lwz 3, L..C6(2) # @sll
+; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    lbz 4, 0(29)
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lbz 3, 0(30)
+; AIX32-NEXT:    li 7, 5
+; AIX32-NEXT:    li 8, 5
+; AIX32-NEXT:    extsb 6, 4
+; AIX32-NEXT:    stw 3, 68(1)
+; AIX32-NEXT:    lwz 3, L..C7(2) # @ull
+; AIX32-NEXT:    addi 4, 1, 64
+; AIX32-NEXT:    stw 31, 64(1)
+; AIX32-NEXT:    srawi 5, 6, 31
+; AIX32-NEXT:    bl .__atomic_compare_exchange_8[PR]
+; AIX32-NEXT:    nop
+; AIX32-NEXT:    stw 3, 0(28)
+; AIX32-NEXT:    lwz 31, 140(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 30, 136(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 29, 132(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 28, 128(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 27, 124(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 26, 120(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 25, 116(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 24, 112(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 23, 108(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 22, 104(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 21, 100(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 20, 96(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 19, 92(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 18, 88(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 17, 84(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 16, 80(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 15, 76(1) # 4-byte Folded Reload
+; AIX32-NEXT:    lwz 14, 72(1) # 4-byte Folded Reload
+; AIX32-NEXT:    addi 1, 1, 144
+; AIX32-NEXT:    lwz 0, 8(1)
+; AIX32-NEXT:    mtlr 0
 ; AIX32-NEXT:    blr
 entry:
   %0 = load i8, ptr @uc, align 1
@@ -5852,20 +5852,23 @@ entry:
 define dso_local i64 @cmpswplp(ptr noundef %ptr, ptr nocapture noundef readnone %oldval, i64 noundef %newval) local_unnamed_addr #0 {
 ; CHECK-LABEL: cmpswplp:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:   ldarx 4, 0, 3
-; CHECK-NEXT:   cmpld   4, 5
-; CHECK-NEXT:   bne     0, .LBB6_2
-; CHECK-NEXT: # %bb.1:                                # %cmpxchg.fencedstore
-; CHECK-NEXT:   addi 4, 5, 1
-; CHECK-NEXT:   stdcx. 4, 0, 3
-; CHECK-NEXT:   beq     0, .LBB6_4
-; CHECK-NEXT: .LBB6_2:                                # %cmpxchg.failure
-; CHECK-NEXT:   crxor 20, 20, 20
-; CHECK-NEXT: .LBB6_3:                                # %cmpxchg.end
-; CHECK-NEXT:   li 3, 66
-; CHECK-NEXT:   li 4, 55
-; CHECK-NEXT:   isel 3, 4, 3, 20
-; CHECK-NEXT:   blr
+; CHECK-NEXT:    ldarx 4, 0, 3
+; CHECK-NEXT:    cmpld 4, 5
+; CHECK-NEXT:    bne 0, .LBB6_2
+; CHECK-NEXT:  # %bb.1: # %cmpxchg.fencedstore
+; CHECK-NEXT:    addi 4, 5, 1
+; CHECK-NEXT:    stdcx. 4, 0, 3
+; CHECK-NEXT:    beq 0, .LBB6_4
+; CHECK-NEXT:  .LBB6_2: # %cmpxchg.failure
+; CHECK-NEXT:    crxor 20, 20, 20
+; CHECK-NEXT:  .LBB6_3: # %cmpxchg.end
+; CHECK-NEXT:    li 3, 66
+; CHECK-NEXT:    li 4, 55
+; CHECK-NEXT:    isel 3, 4, 3, 20
+; CHECK-NEXT:    blr
+; CHECK-NEXT:  .LBB6_4:
+; CHECK-NEXT:    creqv 20, 20, 20
+; CHECK-NEXT:    b .LBB6_3
 ;
 ; AIX32-LABEL: cmpswplp:
 ; AIX32:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/PowerPC/loop-comment.ll b/llvm/test/CodeGen/PowerPC/loop-comment.ll
index 1fa9dda51ef9..34b29cbe901e 100644
--- a/llvm/test/CodeGen/PowerPC/loop-comment.ll
+++ b/llvm/test/CodeGen/PowerPC/loop-comment.ll
@@ -6,18 +6,18 @@ define void @test(ptr %ptr, i8 %cmp, i8 %val) {
 ; PPC64LE:       # %bb.0:
 ; PPC64LE-NEXT:    clrlwi 5, 5, 24
 ; PPC64LE-NEXT:    clrlwi 4, 4, 24
-; PPC64LE-NEXT:    .p2align        5
-; PPC64LE-NEXT:  .LBB0_1:                                # %cmpxchg.start
-; PPC64LE-NEXT:                                          # =>This Inner Loop Header: Depth=1
+; PPC64LE-NEXT:    .p2align 5
+; PPC64LE-NEXT:  .LBB0_1: # %cmpxchg.start
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    lbarx 6, 0, 3
-; PPC64LE-NEXT:    clrlwi  6, 6, 24
+; PPC64LE-NEXT:    clrlwi 6, 6, 24
 ; PPC64LE-NEXT:    cmplw 6, 4
 ; PPC64LE-NEXT:    bnelr 0
-; PPC64LE-NEXT:  # %bb.2:
-; PPC64LE-NEXT:                                          #   in Loop: Header=BB0_1 Depth=1
+; PPC64LE-NEXT:  # %bb.2: # %cmpxchg.fencedstore
+; PPC64LE-NEXT:    #
 ; PPC64LE-NEXT:    stbcx. 5, 0, 3
 ; PPC64LE-NEXT:    bne 0, .LBB0_1
-; PPC64LE-NEXT:  # %bb.3:
+; PPC64LE-NEXT:  # %bb.3: # %cmpxchg.end
 ; PPC64LE-NEXT:    blr
   %res = cmpxchg ptr %ptr, i8 %cmp, i8 %val monotonic monotonic
   ret void
-- 
cgit v1.2.3


From 17f5b8b52a3552de1143efb42af6a94d47d8c7fd Mon Sep 17 00:00:00 2001
From: Andre Kuhlenschmidt <andre.kuhlenschmidt@gmail.com>
Date: Wed, 18 Jun 2025 11:21:35 -0700
Subject: [flang][driver] add ability to look up feature flags without setting
 them (#144559)

This just adds some convenience methods to feature control and rewrites
old code in terms of those methods. Also cleans up some names that I
just realize were overloads of another method.
---
 flang/include/flang/Support/Fortran-features.h | 14 +++++++++++++-
 flang/lib/Frontend/CompilerInvocation.cpp      |  2 +-
 flang/lib/Support/Fortran-features.cpp         | 23 ++++++++++++-----------
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/flang/include/flang/Support/Fortran-features.h b/flang/include/flang/Support/Fortran-features.h
index ea0845b7d605..39356daa3606 100644
--- a/flang/include/flang/Support/Fortran-features.h
+++ b/flang/include/flang/Support/Fortran-features.h
@@ -81,6 +81,9 @@ ENUM_CLASS(UsageWarning, Portability, PointerToUndefinable,
 
 using LanguageFeatures = EnumSet<LanguageFeature, LanguageFeature_enumSize>;
 using UsageWarnings = EnumSet<UsageWarning, UsageWarning_enumSize>;
+using LanguageFeatureOrWarning = std::variant<LanguageFeature, UsageWarning>;
+using LanguageControlFlag =
+    std::pair<LanguageFeatureOrWarning, /*shouldEnable=*/bool>;
 
 class LanguageFeatureControl {
 public:
@@ -94,6 +97,13 @@ public:
   void EnableWarning(UsageWarning w, bool yes = true) {
     warnUsage_.set(w, yes);
   }
+  void EnableWarning(LanguageFeatureOrWarning flag, bool yes = true) {
+    if (std::holds_alternative<LanguageFeature>(flag)) {
+      EnableWarning(std::get<LanguageFeature>(flag), yes);
+    } else {
+      EnableWarning(std::get<UsageWarning>(flag), yes);
+    }
+  }
   void WarnOnAllNonstandard(bool yes = true);
   bool IsWarnOnAllNonstandard() const { return warnAllLanguage_; }
   void WarnOnAllUsage(bool yes = true);
@@ -116,9 +126,11 @@ public:
   bool ShouldWarn(LanguageFeature f) const { return warnLanguage_.test(f); }
   bool ShouldWarn(UsageWarning w) const { return warnUsage_.test(w); }
   // Cli options
+  // Find a warning by its Cli spelling, i.e. '[no-]warning-name'.
+  std::optional<LanguageControlFlag> FindWarning(std::string_view input);
   // Take a string from the Cli and apply it to the LanguageFeatureControl.
   // Return true if the option was recognized (and hence applied).
-  bool ApplyCliOption(std::string input);
+  bool EnableWarning(std::string_view input);
   // The add and replace functions are not currently used but are provided
   // to allow a flexible many-to-one mapping from Cli spellings to enum values.
   // Taking a string by value because the functions own this string after the
diff --git a/flang/lib/Frontend/CompilerInvocation.cpp b/flang/lib/Frontend/CompilerInvocation.cpp
index 147849b0b7d2..2603a3f6dc64 100644
--- a/flang/lib/Frontend/CompilerInvocation.cpp
+++ b/flang/lib/Frontend/CompilerInvocation.cpp
@@ -1011,7 +1011,7 @@ static bool parseDiagArgs(CompilerInvocation &res, llvm::opt::ArgList &args,
       if (wArg == "error") {
         res.setWarnAsErr(true);
         // -W(no-)<feature>
-      } else if (!features.ApplyCliOption(wArg)) {
+      } else if (!features.EnableWarning(wArg)) {
         const unsigned diagID = diags.getCustomDiagID(
             clang::DiagnosticsEngine::Error, "Unknown diagnostic option: -W%0");
         diags.Report(diagID) << wArg;
diff --git a/flang/lib/Support/Fortran-features.cpp b/flang/lib/Support/Fortran-features.cpp
index 08ded173de51..17b5f8368916 100644
--- a/flang/lib/Support/Fortran-features.cpp
+++ b/flang/lib/Support/Fortran-features.cpp
@@ -151,22 +151,23 @@ LanguageFeatureControl::LanguageFeatureControl() {
   warnLanguage_.set(LanguageFeature::NullActualForAllocatable);
 }
 
-// Take a string from the Cli and apply it to the LanguageFeatureControl.
-bool LanguageFeatureControl::ApplyCliOption(std::string input) {
+std::optional<LanguageControlFlag> LanguageFeatureControl::FindWarning(
+    std::string_view input) {
   bool negated{false};
   if (input.size() > 3 && input.substr(0, 3) == "no-") {
     negated = true;
     input = input.substr(3);
   }
-  if (auto it{cliOptions_.find(input)}; it != cliOptions_.end()) {
-    if (std::holds_alternative<LanguageFeature>(it->second)) {
-      EnableWarning(std::get<LanguageFeature>(it->second), !negated);
-      return true;
-    }
-    if (std::holds_alternative<UsageWarning>(it->second)) {
-      EnableWarning(std::get<UsageWarning>(it->second), !negated);
-      return true;
-    }
+  if (auto it{cliOptions_.find(std::string{input})}; it != cliOptions_.end()) {
+    return std::make_pair(it->second, !negated);
+  }
+  return std::nullopt;
+}
+
+bool LanguageFeatureControl::EnableWarning(std::string_view input) {
+  if (auto warningAndEnabled{FindWarning(input)}) {
+    EnableWarning(warningAndEnabled->first, warningAndEnabled->second);
+    return true;
   }
   return false;
 }
-- 
cgit v1.2.3


From 8c3fbaf0ee7322e948403d2234a7230bd6137c98 Mon Sep 17 00:00:00 2001
From: "Walter J.T.V" <81811777+eZWALT@users.noreply.github.com>
Date: Wed, 18 Jun 2025 20:52:41 +0200
Subject: [Clang][OpenMP][LoopTransformations] Fix incorrect number of
 generated loops for Tile and Reverse directives (#140532)

This patch is closely related to #139293 and addresses an existing issue
in the loop transformation codebase. Specifically, it corrects the
handling of the `NumGeneratedLoops` variable in
`OMPLoopTransformationDirective` AST nodes and its inheritors (such as
OMPUnrollDirective, OMPTileDirective, etc.).

Previously, this variable was inaccurately set for certain
transformations like reverse or tile. While this did not lead to
functional bugs, since the value was only checked to determine whether
it was greater than zero or equal to zero, the inconsistency could
introduce problems when supporting more complex directives in the
future.
---
 clang/include/clang/AST/StmtOpenMP.h      | 23 +++++++++++++++--------
 clang/lib/AST/StmtOpenMP.cpp              | 11 +++++++----
 clang/lib/Sema/SemaOpenMP.cpp             |  4 ++--
 clang/lib/Serialization/ASTReaderStmt.cpp |  5 ++---
 4 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 736bcabbad1f..e2fd2114026f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5787,10 +5787,13 @@ class OMPReverseDirective final : public OMPLoopTransformationDirective {
     TransformedStmtOffset,
   };
 
-  explicit OMPReverseDirective(SourceLocation StartLoc, SourceLocation EndLoc)
+  explicit OMPReverseDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
       : OMPLoopTransformationDirective(OMPReverseDirectiveClass,
                                        llvm::omp::OMPD_reverse, StartLoc,
-                                       EndLoc, 1) {}
+                                       EndLoc, NumLoops) {
+    setNumGeneratedLoops(NumLoops);
+  }
 
   void setPreInits(Stmt *PreInits) {
     Data->getChildren()[PreInitsOffset] = PreInits;
@@ -5806,19 +5809,23 @@ public:
   /// \param C         Context of the AST.
   /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
   /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param NumLoops Number of affected loops
   /// \param AssociatedStmt  The outermost associated loop.
   /// \param TransformedStmt The loop nest after tiling, or nullptr in
   ///                        dependent contexts.
   /// \param PreInits   Helper preinits statements for the loop nest.
-  static OMPReverseDirective *
-  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
-         Stmt *AssociatedStmt, Stmt *TransformedStmt, Stmt *PreInits);
+  static OMPReverseDirective *Create(const ASTContext &C,
+                                     SourceLocation StartLoc,
+                                     SourceLocation EndLoc,
+                                     Stmt *AssociatedStmt, unsigned NumLoops,
+                                     Stmt *TransformedStmt, Stmt *PreInits);
 
   /// Build an empty '#pragma omp reverse' AST node for deserialization.
   ///
   /// \param C          Context of the AST.
-  /// \param NumClauses Number of clauses to allocate.
-  static OMPReverseDirective *CreateEmpty(const ASTContext &C);
+  /// \param NumLoops   Number of associated loops to allocate
+  static OMPReverseDirective *CreateEmpty(const ASTContext &C,
+                                          unsigned NumLoops);
 
   /// Gets/sets the associated loops after the transformation, i.e. after
   /// de-sugaring.
@@ -5857,7 +5864,7 @@ class OMPInterchangeDirective final : public OMPLoopTransformationDirective {
       : OMPLoopTransformationDirective(OMPInterchangeDirectiveClass,
                                        llvm::omp::OMPD_interchange, StartLoc,
                                        EndLoc, NumLoops) {
-    setNumGeneratedLoops(3 * NumLoops);
+    setNumGeneratedLoops(NumLoops);
   }
 
   void setPreInits(Stmt *PreInits) {
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 093e1f659916..2eeb5e45ab51 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -471,18 +471,21 @@ OMPUnrollDirective *OMPUnrollDirective::CreateEmpty(const ASTContext &C,
 OMPReverseDirective *
 OMPReverseDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                             SourceLocation EndLoc, Stmt *AssociatedStmt,
-                            Stmt *TransformedStmt, Stmt *PreInits) {
+                            unsigned NumLoops, Stmt *TransformedStmt,
+                            Stmt *PreInits) {
   OMPReverseDirective *Dir = createDirective<OMPReverseDirective>(
-      C, {}, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc);
+      C, {}, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
   Dir->setTransformedStmt(TransformedStmt);
   Dir->setPreInits(PreInits);
   return Dir;
 }
 
-OMPReverseDirective *OMPReverseDirective::CreateEmpty(const ASTContext &C) {
+OMPReverseDirective *OMPReverseDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumLoops) {
   return createEmptyDirective<OMPReverseDirective>(
       C, /*NumClauses=*/0, /*HasAssociatedStmt=*/true,
-      TransformedStmtOffset + 1, SourceLocation(), SourceLocation());
+      TransformedStmtOffset + 1, SourceLocation(), SourceLocation(), NumLoops);
 }
 
 OMPInterchangeDirective *OMPInterchangeDirective::Create(
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index d928b7ae2b4c..00f465818080 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15140,7 +15140,7 @@ StmtResult SemaOpenMP::ActOnOpenMPReverseDirective(Stmt *AStmt,
   // instantiated.
   if (SemaRef.CurContext->isDependentContext())
     return OMPReverseDirective::Create(Context, StartLoc, EndLoc, AStmt,
-                                       nullptr, nullptr);
+                                       NumLoops, nullptr, nullptr);
 
   assert(LoopHelpers.size() == NumLoops &&
          "Expecting a single-dimensional loop iteration space");
@@ -15299,7 +15299,7 @@ StmtResult SemaOpenMP::ActOnOpenMPReverseDirective(Stmt *AStmt,
       ForStmt(Context, Init.get(), Cond.get(), nullptr, Incr.get(),
               ReversedBody, LoopHelper.Init->getBeginLoc(),
               LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
-  return OMPReverseDirective::Create(Context, StartLoc, EndLoc, AStmt,
+  return OMPReverseDirective::Create(Context, StartLoc, EndLoc, AStmt, NumLoops,
                                      ReversedFor,
                                      buildPreInits(Context, PreInits));
 }
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 65102b64030c..44cfb83ad2db 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -3602,11 +3602,10 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
     }
 
     case STMT_OMP_REVERSE_DIRECTIVE: {
-      assert(Record[ASTStmtReader::NumStmtFields] == 1 &&
-             "Reverse directive accepts only a single loop");
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
       assert(Record[ASTStmtReader::NumStmtFields + 1] == 0 &&
              "Reverse directive has no clauses");
-      S = OMPReverseDirective::CreateEmpty(Context);
+      S = OMPReverseDirective::CreateEmpty(Context, NumLoops);
       break;
     }
 
-- 
cgit v1.2.3


From ab6beeca9ccc1968661eea27c1a55e8734f7437b Mon Sep 17 00:00:00 2001
From: uthmanna <114300283+uthmanna@users.noreply.github.com>
Date: Wed, 18 Jun 2025 21:00:10 +0200
Subject: [llvm-cov] Export decision coverage to output json (#144335)

This commit adds decision coverage counts derived from MC/DC test vector execution to the JSON output of llvm-cov, as
discussed here: [Missing Decision Coverage (DC) in output
json](https://discourse.llvm.org/t/missing-decision-coverage-dc-in-output-json/86783)
with @evodius96

---------

Co-authored-by: uthmanna <andre.uthmann@vector.com>
---
 llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h | 12 ++++++++++++
 llvm/tools/llvm-cov/CoverageExporterJson.cpp             |  6 ++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index d1230b0ba7c5..8e6180be25b5 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -31,6 +31,7 @@
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
+#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -494,6 +495,17 @@ public:
     return TV[TestVectorIndex].first[PosToID[Condition]];
   }
 
+  /// Return the number of True and False decisions for all executed test
+  /// vectors.
+  std::pair<unsigned, unsigned> getDecisions() const {
+    const unsigned TrueDecisions =
+        std::count_if(TV.begin(), TV.end(), [](const auto &TestVec) {
+          return TestVec.second == CondState::MCDC_True;
+        });
+
+    return {TrueDecisions, TV.size() - TrueDecisions};
+  }
+
   /// Return the Result evaluation for an executed test vector.
   /// See MCDCRecordProcessor::RecordTestVector().
   CondState getTVResult(unsigned TestVectorIndex) {
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index 4088c1b053aa..024693a24cc2 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -62,7 +62,7 @@
 #include <utility>
 
 /// The semantic version combined as a string.
-#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.1"
+#define LLVM_COVERAGE_EXPORT_JSON_STR "3.0.0"
 
 /// Unique type identifier for JSON coverage export.
 #define LLVM_COVERAGE_EXPORT_JSON_TYPE_STR "llvm.coverage.json.export"
@@ -110,8 +110,10 @@ json::Array gatherConditions(const coverage::MCDCRecord &Record) {
 
 json::Array renderMCDCRecord(const coverage::MCDCRecord &Record) {
   const llvm::coverage::CounterMappingRegion &CMR = Record.getDecisionRegion();
+  const auto [TrueDecisions, FalseDecisions] = Record.getDecisions();
   return json::Array({CMR.LineStart, CMR.ColumnStart, CMR.LineEnd,
-                      CMR.ColumnEnd, CMR.ExpandedFileID, int64_t(CMR.Kind),
+                      CMR.ColumnEnd, TrueDecisions, FalseDecisions,
+                      CMR.ExpandedFileID, int64_t(CMR.Kind),
                       gatherConditions(Record)});
 }
 
-- 
cgit v1.2.3


From ca9a09dbe679dbdd4d47cb7894977e04c3bb914e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Wed, 18 Jun 2025 12:03:17 -0700
Subject: [libc++] Fix a typo in documentation (#144763)

---
 libcxx/docs/ABIGuarantees.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/docs/ABIGuarantees.rst b/libcxx/docs/ABIGuarantees.rst
index e6ac4f2b5b23..c7d5afe1080b 100644
--- a/libcxx/docs/ABIGuarantees.rst
+++ b/libcxx/docs/ABIGuarantees.rst
@@ -92,7 +92,7 @@ Linking TUs which have been compiled with different flags affecting code gen
 There are a lot of compiler (and library) flags which change the code generated for functions. This includes flags like
 ``-O1``, which are guaranteed by the compiler to not change the observable behaviour of a correct program, as well as
 flags like ``-fexceptions``, which **do** change the observable behaviour. libc++ allows linking of TUs which have been
-compiled whith specific flags only and makes no guarantees for any of the flags not listed below.
+compiled with specific flags only and makes no guarantees for any of the flags not listed below.
 
 The flags allowed (in any combination) are:
 - ``-f[no-]exceptions``
-- 
cgit v1.2.3


From a94eb27a29ef3aee5ccafc1d7bebee1c8efbaf38 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Wed, 18 Jun 2025 20:18:22 +0100
Subject: [HashRecognize] Fix big-endian CRC tables (#144754)

Big-endian CRC tables are incorrect due to the initial value of CRC in
genSarwateTable being hard-coded for CRC-8. 128 is the signed-min value
for CRC-8, but it should be generalized to APInt::getSignedMinValue. The
issue was found when writing CRC verification tests for llvm-test-suite.
---
 llvm/lib/Analysis/HashRecognize.cpp                |  2 +-
 .../HashRecognize/cyclic-redundancy-check.ll       | 96 +++++++++++-----------
 2 files changed, 49 insertions(+), 49 deletions(-)

diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 1edb8b3bdc9a..987d13731276 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -478,7 +478,7 @@ CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
   Table[0] = APInt::getZero(BW);
 
   if (ByteOrderSwapped) {
-    APInt CRCInit(BW, 128);
+    APInt CRCInit = APInt::getSignedMinValue(BW);
     for (unsigned I = 1; I < 256; I <<= 1) {
       CRCInit = CRCInit.shl(1) ^
                 (CRCInit.isSignBitSet() ? GenPoly : APInt::getZero(BW));
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 7a3082056ad2..0fbc376c40d7 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -151,22 +151,22 @@ define i16 @crc16.be.tc8.crc.init.li(i16 %checksum, i8 %msg) {
 ; CHECK-NEXT:    Generating polynomial: 4129
 ; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
 ; CHECK-NEXT:    Computed CRC lookup table:
-; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
-; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
-; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
-; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
-; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
-; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
-; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
-; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
-; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
-; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
-; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
-; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
-; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
-; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
-; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
-; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+; CHECK-NEXT:  0 4129 8258 12387 16516 20645 24774 28903 33032 37161 41290 45419 49548 53677 57806 61935
+; CHECK-NEXT:  4657 528 12915 8786 21173 17044 29431 25302 37689 33560 45947 41818 54205 50076 62463 58334
+; CHECK-NEXT:  9314 13379 1056 5121 25830 29895 17572 21637 42346 46411 34088 38153 58862 62927 50604 54669
+; CHECK-NEXT:  13907 9842 5649 1584 30423 26358 22165 18100 46939 42874 38681 34616 63455 59390 55197 51132
+; CHECK-NEXT:  18628 22757 26758 30887 2112 6241 10242 14371 51660 55789 59790 63919 35144 39273 43274 47403
+; CHECK-NEXT:  23285 19156 31415 27286 6769 2640 14899 10770 56317 52188 64447 60318 39801 35672 47931 43802
+; CHECK-NEXT:  27814 31879 19684 23749 11298 15363 3168 7233 60846 64911 52716 56781 44330 48395 36200 40265
+; CHECK-NEXT:  32407 28342 24277 20212 15891 11826 7761 3696 65439 61374 57309 53244 48923 44858 40793 36728
+; CHECK-NEXT:  37256 33193 45514 41451 53516 49453 61774 57711 4224 161 12482 8419 20484 16421 28742 24679
+; CHECK-NEXT:  33721 37784 41979 46042 49981 54044 58239 62302 689 4752 8947 13010 16949 21012 25207 29270
+; CHECK-NEXT:  46570 42443 38312 34185 62830 58703 54572 50445 13538 9411 5280 1153 29798 25671 21540 17413
+; CHECK-NEXT:  42971 47098 34713 38840 59231 63358 50973 55100 9939 14066 1681 5808 26199 30326 17941 22068
+; CHECK-NEXT:  55628 51565 63758 59695 39368 35305 47498 43435 22596 18533 30726 26663 6336 2273 14466 10403
+; CHECK-NEXT:  52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
+; CHECK-NEXT:  64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
+; CHECK-NEXT:  61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
 ;
 entry:
   %msg.ext = zext i8 %msg to i16
@@ -196,22 +196,22 @@ define i16 @crc16.be.tc8.crc.init.arg(i16 %crc.init) {
 ; CHECK-NEXT:    Generating polynomial: 4129
 ; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
 ; CHECK-NEXT:    Computed CRC lookup table:
-; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
-; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
-; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
-; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
-; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
-; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
-; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
-; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
-; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
-; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
-; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
-; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
-; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
-; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
-; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
-; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+; CHECK-NEXT:  0 4129 8258 12387 16516 20645 24774 28903 33032 37161 41290 45419 49548 53677 57806 61935
+; CHECK-NEXT:  4657 528 12915 8786 21173 17044 29431 25302 37689 33560 45947 41818 54205 50076 62463 58334
+; CHECK-NEXT:  9314 13379 1056 5121 25830 29895 17572 21637 42346 46411 34088 38153 58862 62927 50604 54669
+; CHECK-NEXT:  13907 9842 5649 1584 30423 26358 22165 18100 46939 42874 38681 34616 63455 59390 55197 51132
+; CHECK-NEXT:  18628 22757 26758 30887 2112 6241 10242 14371 51660 55789 59790 63919 35144 39273 43274 47403
+; CHECK-NEXT:  23285 19156 31415 27286 6769 2640 14899 10770 56317 52188 64447 60318 39801 35672 47931 43802
+; CHECK-NEXT:  27814 31879 19684 23749 11298 15363 3168 7233 60846 64911 52716 56781 44330 48395 36200 40265
+; CHECK-NEXT:  32407 28342 24277 20212 15891 11826 7761 3696 65439 61374 57309 53244 48923 44858 40793 36728
+; CHECK-NEXT:  37256 33193 45514 41451 53516 49453 61774 57711 4224 161 12482 8419 20484 16421 28742 24679
+; CHECK-NEXT:  33721 37784 41979 46042 49981 54044 58239 62302 689 4752 8947 13010 16949 21012 25207 29270
+; CHECK-NEXT:  46570 42443 38312 34185 62830 58703 54572 50445 13538 9411 5280 1153 29798 25671 21540 17413
+; CHECK-NEXT:  42971 47098 34713 38840 59231 63358 50973 55100 9939 14066 1681 5808 26199 30326 17941 22068
+; CHECK-NEXT:  55628 51565 63758 59695 39368 35305 47498 43435 22596 18533 30726 26663 6336 2273 14466 10403
+; CHECK-NEXT:  52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
+; CHECK-NEXT:  64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
+; CHECK-NEXT:  61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
 ;
 entry:
   br label %loop
@@ -238,22 +238,22 @@ define i16 @crc16.be.tc8.crc.init.arg.flipped.sb.check(i16 %crc.init) {
 ; CHECK-NEXT:    Generating polynomial: 4129
 ; CHECK-NEXT:    Computed CRC: %crc.next = select i1 %check.sb, i16 %crc.shl, i16 %crc.xor
 ; CHECK-NEXT:    Computed CRC lookup table:
-; CHECK-NEXT:  0 256 512 768 1024 1280 1536 1792 2048 2304 2560 2816 3072 3328 3584 3840
-; CHECK-NEXT:  4096 4352 4608 4864 5120 5376 5632 5888 6144 6400 6656 6912 7168 7424 7680 7936
-; CHECK-NEXT:  8192 8448 8704 8960 9216 9472 9728 9984 10240 10496 10752 11008 11264 11520 11776 12032
-; CHECK-NEXT:  12288 12544 12800 13056 13312 13568 13824 14080 14336 14592 14848 15104 15360 15616 15872 16128
-; CHECK-NEXT:  16384 16640 16896 17152 17408 17664 17920 18176 18432 18688 18944 19200 19456 19712 19968 20224
-; CHECK-NEXT:  20480 20736 20992 21248 21504 21760 22016 22272 22528 22784 23040 23296 23552 23808 24064 24320
-; CHECK-NEXT:  24576 24832 25088 25344 25600 25856 26112 26368 26624 26880 27136 27392 27648 27904 28160 28416
-; CHECK-NEXT:  28672 28928 29184 29440 29696 29952 30208 30464 30720 30976 31232 31488 31744 32000 32256 32512
-; CHECK-NEXT:  32768 33024 33280 33536 33792 34048 34304 34560 34816 35072 35328 35584 35840 36096 36352 36608
-; CHECK-NEXT:  36864 37120 37376 37632 37888 38144 38400 38656 38912 39168 39424 39680 39936 40192 40448 40704
-; CHECK-NEXT:  40960 41216 41472 41728 41984 42240 42496 42752 43008 43264 43520 43776 44032 44288 44544 44800
-; CHECK-NEXT:  45056 45312 45568 45824 46080 46336 46592 46848 47104 47360 47616 47872 48128 48384 48640 48896
-; CHECK-NEXT:  49152 49408 49664 49920 50176 50432 50688 50944 51200 51456 51712 51968 52224 52480 52736 52992
-; CHECK-NEXT:  53248 53504 53760 54016 54272 54528 54784 55040 55296 55552 55808 56064 56320 56576 56832 57088
-; CHECK-NEXT:  57344 57600 57856 58112 58368 58624 58880 59136 59392 59648 59904 60160 60416 60672 60928 61184
-; CHECK-NEXT:  61440 61696 61952 62208 62464 62720 62976 63232 63488 63744 64000 64256 64512 64768 65024 65280
+; CHECK-NEXT:  0 4129 8258 12387 16516 20645 24774 28903 33032 37161 41290 45419 49548 53677 57806 61935
+; CHECK-NEXT:  4657 528 12915 8786 21173 17044 29431 25302 37689 33560 45947 41818 54205 50076 62463 58334
+; CHECK-NEXT:  9314 13379 1056 5121 25830 29895 17572 21637 42346 46411 34088 38153 58862 62927 50604 54669
+; CHECK-NEXT:  13907 9842 5649 1584 30423 26358 22165 18100 46939 42874 38681 34616 63455 59390 55197 51132
+; CHECK-NEXT:  18628 22757 26758 30887 2112 6241 10242 14371 51660 55789 59790 63919 35144 39273 43274 47403
+; CHECK-NEXT:  23285 19156 31415 27286 6769 2640 14899 10770 56317 52188 64447 60318 39801 35672 47931 43802
+; CHECK-NEXT:  27814 31879 19684 23749 11298 15363 3168 7233 60846 64911 52716 56781 44330 48395 36200 40265
+; CHECK-NEXT:  32407 28342 24277 20212 15891 11826 7761 3696 65439 61374 57309 53244 48923 44858 40793 36728
+; CHECK-NEXT:  37256 33193 45514 41451 53516 49453 61774 57711 4224 161 12482 8419 20484 16421 28742 24679
+; CHECK-NEXT:  33721 37784 41979 46042 49981 54044 58239 62302 689 4752 8947 13010 16949 21012 25207 29270
+; CHECK-NEXT:  46570 42443 38312 34185 62830 58703 54572 50445 13538 9411 5280 1153 29798 25671 21540 17413
+; CHECK-NEXT:  42971 47098 34713 38840 59231 63358 50973 55100 9939 14066 1681 5808 26199 30326 17941 22068
+; CHECK-NEXT:  55628 51565 63758 59695 39368 35305 47498 43435 22596 18533 30726 26663 6336 2273 14466 10403
+; CHECK-NEXT:  52093 56156 60223 64286 35833 39896 43963 48026 19061 23124 27191 31254 2801 6864 10931 14994
+; CHECK-NEXT:  64814 60687 56684 52557 48554 44427 40424 36297 31782 27655 23652 19525 15522 11395 7392 3265
+; CHECK-NEXT:  61215 65342 53085 57212 44955 49082 36825 40952 28183 32310 20053 24180 11923 16050 3793 7920
 ;
 entry:
   br label %loop
-- 
cgit v1.2.3


From f13b9e3643661ea2cda252c7e2c59ace036407c7 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Wed, 18 Jun 2025 20:18:53 +0100
Subject: [HashRecognize] Don't const-qualify Values in result (#144752)

Const-qualifying Values in the analysis result makes them unusable with
IRBuilder. The issue was discovered when attempting to use the result of
the analysis for a transform.
---
 llvm/include/llvm/Analysis/HashRecognize.h | 12 ++++++------
 llvm/lib/Analysis/HashRecognize.cpp        |  8 ++++----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/llvm/include/llvm/Analysis/HashRecognize.h b/llvm/include/llvm/Analysis/HashRecognize.h
index 8ab68a5dc2cb..c169383bf7b0 100644
--- a/llvm/include/llvm/Analysis/HashRecognize.h
+++ b/llvm/include/llvm/Analysis/HashRecognize.h
@@ -53,7 +53,7 @@ struct PolynomialInfo {
   // division in the case of CRC. Since polynomial division is an XOR in
   // GF(2^m), this variable must be XOR'ed with RHS in a loop to yield the
   // ComputedValue.
-  const Value *LHS;
+  Value *LHS;
 
   // The generating polynomial, or the RHS of the polynomial division in the
   // case of CRC.
@@ -61,7 +61,7 @@ struct PolynomialInfo {
 
   // The final computed value. This is a remainder of a polynomial division in
   // the case of CRC, which must be zero.
-  const Value *ComputedValue;
+  Value *ComputedValue;
 
   // Set to true in the case of big-endian.
   bool ByteOrderSwapped;
@@ -69,11 +69,11 @@ struct PolynomialInfo {
   // An optional auxiliary checksum that augments the LHS. In the case of CRC,
   // it is XOR'ed with the LHS, so that the computation's final remainder is
   // zero.
-  const Value *LHSAux;
+  Value *LHSAux;
 
-  PolynomialInfo(unsigned TripCount, const Value *LHS, const APInt &RHS,
-                 const Value *ComputedValue, bool ByteOrderSwapped,
-                 const Value *LHSAux = nullptr);
+  PolynomialInfo(unsigned TripCount, Value *LHS, const APInt &RHS,
+                 Value *ComputedValue, bool ByteOrderSwapped,
+                 Value *LHSAux = nullptr);
 };
 
 /// The analysis.
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 987d13731276..06a3738018e9 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -442,9 +442,9 @@ getRecurrences(BasicBlock *LoopLatch, const PHINode *IndVar, const Loop &L) {
   return std::make_pair(SimpleRecurrence, ConditionalRecurrence);
 }
 
-PolynomialInfo::PolynomialInfo(unsigned TripCount, const Value *LHS,
-                               const APInt &RHS, const Value *ComputedValue,
-                               bool ByteOrderSwapped, const Value *LHSAux)
+PolynomialInfo::PolynomialInfo(unsigned TripCount, Value *LHS, const APInt &RHS,
+                               Value *ComputedValue, bool ByteOrderSwapped,
+                               Value *LHSAux)
     : TripCount(TripCount), LHS(LHS), RHS(RHS), ComputedValue(ComputedValue),
       ByteOrderSwapped(ByteOrderSwapped), LHSAux(LHSAux) {}
 
@@ -623,7 +623,7 @@ HashRecognize::recognizeCRC() const {
   if (!checkExtractBits(ResultBits, TC, IsZero, *ByteOrderSwapped))
     return ErrBits(ResultBits, TC, *ByteOrderSwapped);
 
-  const Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
+  Value *LHSAux = SimpleRecurrence ? SimpleRecurrence.Start : nullptr;
   return PolynomialInfo(TC, ConditionalRecurrence.Start, GenPoly, ComputedValue,
                         *ByteOrderSwapped, LHSAux);
 }
-- 
cgit v1.2.3


From 156a64c585faf0870936b62ec85fae19ceb9ad3f Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Wed, 18 Jun 2025 20:19:25 +0100
Subject: [HashRecognize] Tighten pre-conditions for analysis (#144757)

Exit early if the TC is not a byte-multiple, as optimization works by
dividing TC by 8. Also delay the SCEV TC query.
---
 llvm/lib/Analysis/HashRecognize.cpp                |  8 +--
 .../HashRecognize/cyclic-redundancy-check.ll       | 62 ++++++++++++++++++++--
 2 files changed, 61 insertions(+), 9 deletions(-)

diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index 06a3738018e9..d11602f92187 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -561,14 +561,14 @@ std::variant<PolynomialInfo, ErrBits, StringRef>
 HashRecognize::recognizeCRC() const {
   if (!L.isInnermost())
     return "Loop is not innermost";
-  unsigned TC = SE.getSmallConstantMaxTripCount(&L);
-  if (!TC || TC > 256)
-    return "Unable to find a small constant trip count";
   BasicBlock *Latch = L.getLoopLatch();
   BasicBlock *Exit = L.getExitBlock();
   const PHINode *IndVar = L.getCanonicalInductionVariable();
-  if (!Latch || !Exit || !IndVar)
+  if (!Latch || !Exit || !IndVar || L.getNumBlocks() != 1)
     return "Loop not in canonical form";
+  unsigned TC = SE.getSmallConstantTripCount(&L);
+  if (!TC || TC > 256 || TC % 8)
+    return "Unable to find a small constant byte-multiple trip count";
 
   auto R = getRecurrences(Latch, IndVar, L);
   if (!R)
diff --git a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
index 0fbc376c40d7..0366684a13b5 100644
--- a/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
+++ b/llvm/test/Analysis/HashRecognize/cyclic-redundancy-check.ll
@@ -384,7 +384,7 @@ exit:                                              ; preds = %loop
 define i16 @not.crc.non.const.tc(i16 %crc.init, i32 %loop.limit) {
 ; CHECK-LABEL: 'not.crc.non.const.tc'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Unable to find a small constant trip count
+; CHECK-NEXT:  Reason: Unable to find a small constant byte-multiple trip count
 ;
 entry:
   br label %loop
@@ -404,8 +404,31 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
-define i16 @not.crc.non.canonical.loop(i16 %crc.init) {
-; CHECK-LABEL: 'not.crc.non.canonical.loop'
+define i16 @not.crc.non.canonical.not.multiple.8(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.non.canonical.not.multiple.8'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Unable to find a small constant byte-multiple trip count
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %loop ]
+  %crc.shl = shl i16 %crc, 1
+  %crc.xor = xor i16 %crc.shl, 4129
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.next = select i1 %check.sb, i16 %crc.xor, i16 %crc.shl
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign eq i32 %iv, 3
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
+define i16 @not.crc.non.canonical.loop.countdown(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.non.canonical.loop.countdown'
 ; CHECK-NEXT:  Did not find a hash algorithm
 ; CHECK-NEXT:  Reason: Loop not in canonical form
 ;
@@ -427,10 +450,39 @@ exit:                                              ; preds = %loop
   ret i16 %crc.next
 }
 
+define i16 @not.crc.non.canonical.loop.multiple.blocks(i16 %crc.init) {
+; CHECK-LABEL: 'not.crc.non.canonical.loop.multiple.blocks'
+; CHECK-NEXT:  Did not find a hash algorithm
+; CHECK-NEXT:  Reason: Loop not in canonical form
+;
+entry:
+  br label %loop
+
+loop:                                              ; preds = %loop, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %continue ]
+  %crc = phi i16 [ %crc.init, %entry ], [ %crc.next, %continue ]
+  %check.sb = icmp slt i16 %crc, 0
+  %crc.shl = shl i16 %crc, 1
+  br i1 %check.sb, label %xor, label %continue
+
+xor:
+  %crc.xor = xor i16 %crc.shl, 4129
+  br label %continue
+
+continue:
+  %crc.next = phi i16 [ %crc.xor, %xor ], [ %crc.shl, %loop ]
+  %iv.next = add nuw nsw i32 %iv, 1
+  %exit.cond = icmp samesign eq i32 %iv, 7
+  br i1 %exit.cond, label %exit, label %loop
+
+exit:                                              ; preds = %loop
+  ret i16 %crc.next
+}
+
 define i16 @not.crc.tc.limit(i16 %crc.init) {
 ; CHECK-LABEL: 'not.crc.tc.limit'
 ; CHECK-NEXT:  Did not find a hash algorithm
-; CHECK-NEXT:  Reason: Unable to find a small constant trip count
+; CHECK-NEXT:  Reason: Unable to find a small constant byte-multiple trip count
 ;
 entry:
   br label %loop
@@ -617,7 +669,7 @@ loop:                                              ; preds = %loop, %entry
   %crc.xor = xor i16 %crc.lshr, -24575
   %crc.next = select i1 %check.sb, i16 %crc.lshr, i16 %crc.xor
   %iv.next = add nuw nsw i8 %iv, 1
-  %exit.cond = icmp samesign ult i8 %iv, 20
+  %exit.cond = icmp samesign ult i8 %iv, 31
   br i1 %exit.cond, label %loop, label %exit
 
 exit:                                              ; preds = %loop
-- 
cgit v1.2.3


From 88d250729eb00842a41c946632bcacf1af106f64 Mon Sep 17 00:00:00 2001
From: Alan Phipps <a-phipps@ti.com>
Date: Wed, 18 Jun 2025 14:33:59 -0500
Subject: Revert "[llvm-cov] Export decision coverage to output json" (#144783)

Reverts llvm/llvm-project#144335

Need to resolve test failures
---
 llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h | 12 ------------
 llvm/tools/llvm-cov/CoverageExporterJson.cpp             |  6 ++----
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index 8e6180be25b5..d1230b0ba7c5 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -31,7 +31,6 @@
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
-#include <algorithm>
 #include <cassert>
 #include <cstdint>
 #include <iterator>
@@ -495,17 +494,6 @@ public:
     return TV[TestVectorIndex].first[PosToID[Condition]];
   }
 
-  /// Return the number of True and False decisions for all executed test
-  /// vectors.
-  std::pair<unsigned, unsigned> getDecisions() const {
-    const unsigned TrueDecisions =
-        std::count_if(TV.begin(), TV.end(), [](const auto &TestVec) {
-          return TestVec.second == CondState::MCDC_True;
-        });
-
-    return {TrueDecisions, TV.size() - TrueDecisions};
-  }
-
   /// Return the Result evaluation for an executed test vector.
   /// See MCDCRecordProcessor::RecordTestVector().
   CondState getTVResult(unsigned TestVectorIndex) {
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index 024693a24cc2..4088c1b053aa 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -62,7 +62,7 @@
 #include <utility>
 
 /// The semantic version combined as a string.
-#define LLVM_COVERAGE_EXPORT_JSON_STR "3.0.0"
+#define LLVM_COVERAGE_EXPORT_JSON_STR "2.0.1"
 
 /// Unique type identifier for JSON coverage export.
 #define LLVM_COVERAGE_EXPORT_JSON_TYPE_STR "llvm.coverage.json.export"
@@ -110,10 +110,8 @@ json::Array gatherConditions(const coverage::MCDCRecord &Record) {
 
 json::Array renderMCDCRecord(const coverage::MCDCRecord &Record) {
   const llvm::coverage::CounterMappingRegion &CMR = Record.getDecisionRegion();
-  const auto [TrueDecisions, FalseDecisions] = Record.getDecisions();
   return json::Array({CMR.LineStart, CMR.ColumnStart, CMR.LineEnd,
-                      CMR.ColumnEnd, TrueDecisions, FalseDecisions,
-                      CMR.ExpandedFileID, int64_t(CMR.Kind),
+                      CMR.ColumnEnd, CMR.ExpandedFileID, int64_t(CMR.Kind),
                       gatherConditions(Record)});
 }
 
-- 
cgit v1.2.3


From fb0651959b1b6ae64f84cf5840adc95923af991f Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Wed, 18 Jun 2025 15:37:56 -0400
Subject: [AArch64] fix trampoline implementation: actually use X15 (#143892)

A incorrect switch statement caused it to try to use X4 instead of X15
in #126743, which would have not worked.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp |  3 +--
 llvm/test/CodeGen/AArch64/trampoline.ll         | 12 ++++++------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index d8b574719dae..581f15277602 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7138,8 +7138,7 @@ SDValue AArch64TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   switch (CC) {
   default:
     NestReg = 0x0f; // X15
-    LLVM_FALLTHROUGH;
-  case CallingConv::ARM64EC_Thunk_Native:
+    break;
   case CallingConv::ARM64EC_Thunk_X64:
     // Must be kept in sync with AArch64CallingConv.td
     NestReg = 0x04; // X4
diff --git a/llvm/test/CodeGen/AArch64/trampoline.ll b/llvm/test/CodeGen/AArch64/trampoline.ll
index d9016b02a0f8..0e682704afbf 100644
--- a/llvm/test/CodeGen/AArch64/trampoline.ll
+++ b/llvm/test/CodeGen/AArch64/trampoline.ll
@@ -83,7 +83,7 @@ define i64 @func1() {
 ; CHECK-LINUX-NEXT:    str w9, [sp, #16]
 ; CHECK-LINUX-NEXT:    add x9, sp, #56
 ; CHECK-LINUX-NEXT:    stp x9, x8, [sp, #24]
-; CHECK-LINUX-NEXT:    mov x8, #132 // =0x84
+; CHECK-LINUX-NEXT:    mov x8, #143 // =0x8f
 ; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-LINUX-NEXT:    movk x8, #177, lsl #32
 ; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #48
@@ -112,7 +112,7 @@ define i64 @func1() {
 ; CHECK-PC-NEXT:    add x0, sp, #8
 ; CHECK-PC-NEXT:    movk w8, #54815, lsl #16
 ; CHECK-PC-NEXT:    str w8, [sp, #16]
-; CHECK-PC-NEXT:    mov x8, #132 // =0x84
+; CHECK-PC-NEXT:    mov x8, #143 // =0x8f
 ; CHECK-PC-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-PC-NEXT:    movk x8, #177, lsl #32
 ; CHECK-PC-NEXT:    movk x8, #22528, lsl #48
@@ -148,7 +148,7 @@ define i64 @func1() {
 ; CHECK-APPLE-NEXT:    mov x0, sp
 ; CHECK-APPLE-NEXT:    movk w8, #54815, lsl #16
 ; CHECK-APPLE-NEXT:    str w8, [sp, #8]
-; CHECK-APPLE-NEXT:    mov x8, #132 ; =0x84
+; CHECK-APPLE-NEXT:    mov x8, #143 ; =0x8f
 ; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-APPLE-NEXT:    movk x8, #177, lsl #32
 ; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #48
@@ -184,7 +184,7 @@ define i64 @func2() {
 ; CHECK-LINUX-NEXT:    add x9, sp, #8
 ; CHECK-LINUX-NEXT:    add x1, x0, #12
 ; CHECK-LINUX-NEXT:    stp x9, x8, [x0, #16]
-; CHECK-LINUX-NEXT:    mov x8, #132 // =0x84
+; CHECK-LINUX-NEXT:    mov x8, #143 // =0x8f
 ; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-LINUX-NEXT:    movk x8, #177, lsl #32
 ; CHECK-LINUX-NEXT:    movk x8, #22528, lsl #48
@@ -210,7 +210,7 @@ define i64 @func2() {
 ; CHECK-PC-NEXT:    mov w8, #544 // =0x220
 ; CHECK-PC-NEXT:    movk w8, #54815, lsl #16
 ; CHECK-PC-NEXT:    str w8, [x0, #8]
-; CHECK-PC-NEXT:    mov x8, #132 // =0x84
+; CHECK-PC-NEXT:    mov x8, #143 // =0x8f
 ; CHECK-PC-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-PC-NEXT:    movk x8, #177, lsl #32
 ; CHECK-PC-NEXT:    movk x8, #22528, lsl #48
@@ -246,7 +246,7 @@ define i64 @func2() {
 ; CHECK-APPLE-NEXT:    mov w8, #544 ; =0x220
 ; CHECK-APPLE-NEXT:    movk w8, #54815, lsl #16
 ; CHECK-APPLE-NEXT:    str w8, [x0, #8]
-; CHECK-APPLE-NEXT:    mov x8, #132 ; =0x84
+; CHECK-APPLE-NEXT:    mov x8, #143 ; =0x8f
 ; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #16
 ; CHECK-APPLE-NEXT:    movk x8, #177, lsl #32
 ; CHECK-APPLE-NEXT:    movk x8, #22528, lsl #48
-- 
cgit v1.2.3


From c04fc5596ec8c197c75b92a086c31438bfb08faf Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Wed, 18 Jun 2025 15:38:34 -0400
Subject: [MemCpyOpt] allow some undef contents overread in
 processMemCpyMemCpyDependence (#143745)

Allows memcpy to memcpy forwarding in cases where the second memcpy is
larger, but the overread is known to be undef, by shrinking the memcpy
size.

Refs https://github.com/llvm/llvm-project/pull/140954 which laid some of
the groundwork for this.
---
 llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp     | 52 +++++++++++++++-------
 .../Transforms/MemCpyOpt/memcpy-memcpy-offset.ll   | 33 +++++++++++---
 .../MemCpyOpt/variable-sized-memcpy-memcpy.ll      | 37 ++++++++++++++-
 3 files changed, 98 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 1c4ec6aa08b4..2b0e221f341e 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -107,6 +107,9 @@ struct MemsetRange {
 
 } // end anonymous namespace
 
+static bool overreadUndefContents(MemorySSA *MSSA, MemCpyInst *MemCpy,
+                                  MemIntrinsic *MemSrc, BatchAAResults &BAA);
+
 bool MemsetRange::isProfitableToUseMemset(const DataLayout &DL) const {
   // If we found more than 4 stores to merge or 16 bytes, use memset.
   if (TheStores.size() >= 4 || End - Start >= 16)
@@ -1129,14 +1132,29 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
     MForwardOffset = *Offset;
   }
 
-  // The length of the memcpy's must be the same, or the preceding one
-  // must be larger than the following one.
-  if (MForwardOffset != 0 || MDep->getLength() != M->getLength()) {
+  Value *CopyLength = M->getLength();
+
+  // The length of the memcpy's must be the same, or the preceding one must be
+  // larger than the following one, or the contents of the overread must be
+  // undefined bytes of a defined size.
+  if (MForwardOffset != 0 || MDep->getLength() != CopyLength) {
     auto *MDepLen = dyn_cast<ConstantInt>(MDep->getLength());
-    auto *MLen = dyn_cast<ConstantInt>(M->getLength());
-    if (!MDepLen || !MLen ||
-        MDepLen->getZExtValue() < MLen->getZExtValue() + MForwardOffset)
+    auto *MLen = dyn_cast<ConstantInt>(CopyLength);
+    // This could be converted to a runtime test (%CopyLength =
+    // min(max(0, MDepLen - MForwardOffset), MLen)), but it is
+    // unclear if that is useful
+    if (!MDepLen || !MLen)
       return false;
+    if (MDepLen->getZExtValue() < MLen->getZExtValue() + MForwardOffset) {
+      if (!overreadUndefContents(MSSA, M, MDep, BAA))
+        return false;
+      if (MDepLen->getZExtValue() <= (uint64_t)MForwardOffset)
+        return false; // Should not reach here (there is obviously no aliasing
+                      // with MDep), so just bail in case it had incomplete info
+                      // somehow
+      CopyLength = ConstantInt::get(CopyLength->getType(),
+                                    MDepLen->getZExtValue() - MForwardOffset);
+    }
   }
 
   IRBuilder<> Builder(M);
@@ -1152,9 +1170,13 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
       eraseInstruction(NewCopySource);
   });
   MaybeAlign CopySourceAlign = MDep->getSourceAlign();
-  // We just need to calculate the actual size of the copy.
-  auto MCopyLoc = MemoryLocation::getForSource(MDep).getWithNewSize(
-      MemoryLocation::getForSource(M).Size);
+  auto MCopyLoc = MemoryLocation::getForSource(MDep);
+  // Truncate the size of the MDep access to just the bytes read
+  if (MDep->getLength() != CopyLength) {
+    auto *ConstLength = cast<ConstantInt>(CopyLength);
+    MCopyLoc = MCopyLoc.getWithNewSize(
+        LocationSize::precise(ConstLength->getZExtValue()));
+  }
 
   // When the forwarding offset is greater than 0, we transform
   //    memcpy(d1 <- s1)
@@ -1223,20 +1245,18 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
   // example we could be moving from movaps -> movq on x86.
   Instruction *NewM;
   if (UseMemMove)
-    NewM =
-        Builder.CreateMemMove(M->getDest(), M->getDestAlign(), CopySource,
-                              CopySourceAlign, M->getLength(), M->isVolatile());
+    NewM = Builder.CreateMemMove(M->getDest(), M->getDestAlign(), CopySource,
+                                 CopySourceAlign, CopyLength, M->isVolatile());
   else if (M->isForceInlined())
     // llvm.memcpy may be promoted to llvm.memcpy.inline, but the converse is
     // never allowed since that would allow the latter to be lowered as a call
     // to an external function.
     NewM = Builder.CreateMemCpyInline(M->getDest(), M->getDestAlign(),
-                                      CopySource, CopySourceAlign,
-                                      M->getLength(), M->isVolatile());
+                                      CopySource, CopySourceAlign, CopyLength,
+                                      M->isVolatile());
   else
     NewM = Builder.CreateMemCpy(M->getDest(), M->getDestAlign(), CopySource,
-                                CopySourceAlign, M->getLength(),
-                                M->isVolatile());
+                                CopySourceAlign, CopyLength, M->isVolatile());
 
   NewM->copyMetadata(*M, LLVMContext::MD_DIAssignID);
 
diff --git a/llvm/test/Transforms/MemCpyOpt/memcpy-memcpy-offset.ll b/llvm/test/Transforms/MemCpyOpt/memcpy-memcpy-offset.ll
index da654438d7bd..7dc579aad02f 100644
--- a/llvm/test/Transforms/MemCpyOpt/memcpy-memcpy-offset.ll
+++ b/llvm/test/Transforms/MemCpyOpt/memcpy-memcpy-offset.ll
@@ -134,14 +134,15 @@ define void @forward_offset_memcpy_inline(ptr %src, ptr %dest) {
   ret void
 }
 
-; We cannot forward `memcpy` because it exceeds the size of `memcpy` it depends on.
-define void @do_not_forward_oversize_offset(ptr %src, ptr %dest) {
-; CHECK-LABEL: define void @do_not_forward_oversize_offset(
+; We can forward `memcpy` by shrinking it to the size of the `memcpy` it depends on.
+define void @forward_oversize_offset(ptr %src, ptr %dest) {
+; CHECK-LABEL: define void @forward_oversize_offset(
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) {
-; CHECK-NEXT:    [[DEP_DEST:%.*]] = alloca [9 x i8], align 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 6, i1 false)
-; CHECK-NEXT:    [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP_OFFSET]], i64 6, i1 false)
+; CHECK-NEXT:    [[CPY_TMP:%.*]] = alloca [9 x i8], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[CPY_TMP]], ptr align 1 [[SRC]], i64 6, i1 false)
+; CHECK-NEXT:    [[CPY_TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[CPY_TMP]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 5, i1 false)
 ; CHECK-NEXT:    ret void
 ;
   %cpy_tmp = alloca %buf, align 1
@@ -214,6 +215,24 @@ define void @pr98675(ptr noalias %p1, ptr noalias %p2) {
   ret void
 }
 
+define void @over_offset_cpy(ptr %src) {
+; CHECK-LABEL: define void @over_offset_cpy(
+; CHECK-SAME: ptr [[SRC:%.*]]) {
+; CHECK-NEXT:    [[TMP:%.*]] = alloca [2 x i8], align 1
+; CHECK-NEXT:    [[DST:%.*]] = alloca i8, align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP]], ptr align 8 [[SRC]], i64 1, i1 false)
+; CHECK-NEXT:    [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 1
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca [2 x i8]
+  %dst = alloca i8
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 1, i1 false)
+  %tmp_offset = getelementptr inbounds i8, ptr %tmp, i64 1
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp_offset, i64 1, i1 false)
+
+  ret void
+}
+
 declare void @use(ptr)
 
 declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1)
diff --git a/llvm/test/Transforms/MemCpyOpt/variable-sized-memcpy-memcpy.ll b/llvm/test/Transforms/MemCpyOpt/variable-sized-memcpy-memcpy.ll
index 4f6b734ec057..95402a8ea686 100644
--- a/llvm/test/Transforms/MemCpyOpt/variable-sized-memcpy-memcpy.ll
+++ b/llvm/test/Transforms/MemCpyOpt/variable-sized-memcpy-memcpy.ll
@@ -18,7 +18,42 @@ define void @test(ptr %src, i64 %size) {
   ret void
 }
 
-; Differing sizes, so left as it is.
+define void @dynalloca_test(ptr %src, i64 %size1) {
+; CHECK-LABEL: @dynalloca_test(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i8, i64 [[SIZE1:%.*]], align 1
+; CHECK-NEXT:    [[DST:%.*]] = alloca i8, i64 [[SIZE1]], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP]], ptr align 8 [[SRC:%.*]], i64 31, i1 false)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST]], ptr align 8 [[SRC]], i64 31, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca i8, i64 %size1
+  %dst = alloca i8, i64 %size1
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 31, i1 false)
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp, i64 32, i1 false)
+
+  ret void
+}
+
+define void @dynalloca_offset_test(ptr %src, i64 %size1) {
+; CHECK-LABEL: @dynalloca_offset_test(
+; CHECK-NEXT:    [[TMP:%.*]] = alloca i8, i64 [[SIZE1:%.*]], align 1
+; CHECK-NEXT:    [[DST:%.*]] = alloca i8, i64 [[SIZE1]], align 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TMP]], ptr align 8 [[SRC:%.*]], i64 31, i1 false)
+; CHECK-NEXT:    [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST]], ptr align 1 [[TMP1]], i64 30, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %tmp = alloca i8, i64 %size1
+  %dst = alloca i8, i64 %size1
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %tmp, ptr align 8 %src, i64 31, i1 false)
+  %tmp_offset = getelementptr inbounds i8, ptr %tmp, i64 1
+  call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst, ptr align 8 %tmp_offset, i64 31, i1 false)
+
+  ret void
+}
+
+; Dynamic sizes, so left as it is.
 define void @negative_test(ptr %src, i64 %size1, i64 %size2) {
 ; CHECK-LABEL: @negative_test(
 ; CHECK-NEXT:    [[TMP:%.*]] = alloca i8, i64 [[SIZE1:%.*]], align 1
-- 
cgit v1.2.3


From 67c52aacae2aa698eb1d31d81d2376bd77723d3a Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Wed, 18 Jun 2025 21:47:50 +0200
Subject: [CIR] Upstream support for IncompleteArrayType (#144138)

This change adds the basic support for IncompleteArray

Issue https://github.com/llvm/llvm-project/issues/130197
---
 clang/lib/CIR/CodeGen/CIRGenTypes.cpp | 16 ++++++++++++++++
 clang/test/CIR/CodeGen/struct.c       | 14 ++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
index bab47924dd71..621eb66962bf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenTypes.cpp
@@ -416,6 +416,22 @@ mlir::Type CIRGenTypes::convertType(QualType type) {
     break;
   }
 
+  case Type::IncompleteArray: {
+    const IncompleteArrayType *arrTy = cast<IncompleteArrayType>(ty);
+    if (arrTy->getIndexTypeCVRQualifiers() != 0)
+      cgm.errorNYI(SourceLocation(), "non trivial array types", type);
+
+    mlir::Type elemTy = convertTypeForMem(arrTy->getElementType());
+    // int X[] -> [0 x int], unless the element type is not sized.  If it is
+    // unsized (e.g. an incomplete record) just use [0 x i8].
+    if (!builder.isSized(elemTy)) {
+      elemTy = cgm.SInt8Ty;
+    }
+
+    resultType = cir::ArrayType::get(elemTy, 0);
+    break;
+  }
+
   case Type::ConstantArray: {
     const ConstantArrayType *arrTy = cast<ConstantArrayType>(ty);
     mlir::Type elemTy = convertTypeForMem(arrTy->getElementType());
diff --git a/clang/test/CIR/CodeGen/struct.c b/clang/test/CIR/CodeGen/struct.c
index ed84edd97e5d..b722b64eeb58 100644
--- a/clang/test/CIR/CodeGen/struct.c
+++ b/clang/test/CIR/CodeGen/struct.c
@@ -19,6 +19,7 @@
 // CIR-DAG: !rec_CycleEnd = !cir.record<struct "CycleEnd" {!cir.ptr<!cir.record<struct "CycleStart" {!cir.ptr<!cir.record<struct "CycleMiddle" {!cir.ptr<!cir.record<struct "CycleEnd">>}>>}>>}>
 // CIR-DAG: !rec_CycleMiddle = !cir.record<struct "CycleMiddle" {!cir.ptr<!rec_CycleEnd>}>
 // CIR-DAG: !rec_CycleStart = !cir.record<struct "CycleStart" {!cir.ptr<!rec_CycleMiddle>}>
+// CIR-DAG: !rec_IncompleteArray = !cir.record<struct "IncompleteArray" {!cir.array<!s32i x 0>}>
 // LLVM-DAG: %struct.CompleteS = type { i32, i8 }
 // LLVM-DAG: %struct.OuterS = type { %struct.InnerS, i32 }
 // LLVM-DAG: %struct.InnerS = type { i32, i8 }
@@ -30,6 +31,7 @@
 // LLVM-DAG: %struct.CycleStart = type { ptr }
 // LLVM-DAG: %struct.CycleMiddle = type { ptr }
 // LLVM-DAG: %struct.CycleEnd = type { ptr }
+// LLVM-DAG: %struct.IncompleteArray = type { [0 x i32] }
 // OGCG-DAG: %struct.CompleteS = type { i32, i8 }
 // OGCG-DAG: %struct.OuterS = type { %struct.InnerS, i32 }
 // OGCG-DAG: %struct.InnerS = type { i32, i8 }
@@ -41,6 +43,7 @@
 // OGCG-DAG: %struct.CycleStart = type { ptr }
 // OGCG-DAG: %struct.CycleMiddle = type { ptr }
 // OGCG-DAG: %struct.CycleEnd = type { ptr }
+// OGCG-DAG: %struct.IncompleteArray = type { [0 x i32] }
 
 struct CompleteS {
   int a;
@@ -149,6 +152,16 @@ struct CycleEnd {
 // LLVM-DAG:  @end = global %struct.CycleEnd zeroinitializer
 // OGCG-DAG:  @end = global %struct.CycleEnd zeroinitializer
 
+struct IncompleteArray {
+  int array[];
+} incomplete;
+
+// CIR: cir.global external @incomplete = #cir.zero : !rec_IncompleteArray
+
+// LLVM-DAG: global %struct.IncompleteArray zeroinitializer
+
+// OGCG-DAG: global %struct.IncompleteArray zeroinitializer
+
 void f(void) {
   struct IncompleteS *p;
 }
@@ -313,3 +326,4 @@ void f6(struct CycleStart *start) {
 // OGCG:   %[[MIDDLE:.*]] = getelementptr inbounds nuw %struct.CycleStart, ptr %{{.*}}, i32 0, i32 0
 // OGCG:   %[[END:.*]] = getelementptr inbounds nuw %struct.CycleMiddle, ptr %{{.*}}, i32 0, i32 0
 // OGCG:   %[[START2:.*]] = getelementptr inbounds nuw %struct.CycleEnd, ptr %{{.*}}, i32 0, i32 0
+
-- 
cgit v1.2.3


From d4b7c0d8b437f50ea254d814a1aeecf87a17be91 Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Wed, 18 Jun 2025 20:49:55 +0100
Subject: [Remarks] Auto-detect remark parser format (#144554)

Add remark format 'Auto', which performs automatic detection of the
remark format using the magic numbers at the beginning of the remarks
files.

The RemarkLinker already did something similar, so we streamlined this
and exposed this to llvm-remarkutil.
---
 llvm/include/llvm/Remarks/RemarkFormat.h            |  5 ++++-
 llvm/include/llvm/Remarks/RemarkLinker.h            |  5 ++---
 llvm/lib/Remarks/RemarkFormat.cpp                   | 18 +++++++++++++++++-
 llvm/lib/Remarks/RemarkLinker.cpp                   | 14 +++-----------
 llvm/lib/Remarks/RemarkParser.cpp                   | 21 +++++++++++++++------
 llvm/lib/Remarks/RemarkSerializer.cpp               |  6 ++++--
 .../Inputs/broken-remark-magic.bitstream            |  1 +
 .../tools/llvm-remarkutil/annotation-count.test     |  2 ++
 .../broken-bitstream-remark-magic.test              |  6 ++++++
 llvm/test/tools/llvm-remarkutil/empty-file.test     |  5 +++++
 .../tools/llvm-remarkutil/instruction-count.test    |  4 +++-
 .../test/tools/llvm-remarkutil/instruction-mix.test |  4 +++-
 .../llvm-remarkutil/size-diff/no-difference.test    |  3 +++
 llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h      |  9 ++++++---
 llvm/unittests/Remarks/RemarksLinkingTest.cpp       |  6 ++----
 15 files changed, 76 insertions(+), 33 deletions(-)
 create mode 100644 llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream
 create mode 100644 llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test

diff --git a/llvm/include/llvm/Remarks/RemarkFormat.h b/llvm/include/llvm/Remarks/RemarkFormat.h
index a39a013dcf90..eda201d4ee6f 100644
--- a/llvm/include/llvm/Remarks/RemarkFormat.h
+++ b/llvm/include/llvm/Remarks/RemarkFormat.h
@@ -23,7 +23,7 @@ namespace remarks {
 constexpr StringLiteral Magic("REMARKS");
 
 /// The format used for serializing/deserializing remarks.
-enum class Format { Unknown, YAML, Bitstream };
+enum class Format { Unknown, Auto, YAML, Bitstream };
 
 /// Parse and validate a string for the remark format.
 LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
@@ -31,6 +31,9 @@ LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
 /// Parse and validate a magic number to a remark format.
 LLVM_ABI Expected<Format> magicToFormat(StringRef Magic);
 
+/// Detect format based on selected format and magic number
+LLVM_ABI Expected<Format> detectFormat(Format Selected, StringRef Magic);
+
 } // end namespace remarks
 } // end namespace llvm
 
diff --git a/llvm/include/llvm/Remarks/RemarkLinker.h b/llvm/include/llvm/Remarks/RemarkLinker.h
index 5343c6214470..67208f40592a 100644
--- a/llvm/include/llvm/Remarks/RemarkLinker.h
+++ b/llvm/include/llvm/Remarks/RemarkLinker.h
@@ -80,13 +80,12 @@ public:
   /// \p Buffer.
   /// \p Buffer can be either a standalone remark container or just
   /// metadata. This takes care of uniquing and merging the remarks.
-  LLVM_ABI Error link(StringRef Buffer,
-                      std::optional<Format> RemarkFormat = std::nullopt);
+  LLVM_ABI Error link(StringRef Buffer, Format RemarkFormat = Format::Auto);
 
   /// Link the remarks found in \p Obj by looking for the right section and
   /// calling the method above.
   LLVM_ABI Error link(const object::ObjectFile &Obj,
-                      std::optional<Format> RemarkFormat = std::nullopt);
+                      Format RemarkFormat = Format::Auto);
 
   /// Serialize the linked remarks to the stream \p OS, using the format \p
   /// RemarkFormat.
diff --git a/llvm/lib/Remarks/RemarkFormat.cpp b/llvm/lib/Remarks/RemarkFormat.cpp
index 800f5bffe70d..1c52e352f939 100644
--- a/llvm/lib/Remarks/RemarkFormat.cpp
+++ b/llvm/lib/Remarks/RemarkFormat.cpp
@@ -42,6 +42,22 @@ Expected<Format> llvm::remarks::magicToFormat(StringRef MagicStr) {
 
   if (Result == Format::Unknown)
     return createStringError(std::make_error_code(std::errc::invalid_argument),
-                             "Unknown remark magic: '%s'", MagicStr.data());
+                             "Automatic detection of remark format failed. "
+                             "Unknown magic number: '%.4s'",
+                             MagicStr.data());
   return Result;
 }
+
+Expected<Format> llvm::remarks::detectFormat(Format Selected,
+                                             StringRef MagicStr) {
+  if (Selected == Format::Unknown)
+    return createStringError(std::make_error_code(std::errc::invalid_argument),
+                             "Unknown remark parser format.");
+  if (Selected != Format::Auto)
+    return Selected;
+
+  // Empty files are valid bitstream files
+  if (MagicStr.empty())
+    return Format::Bitstream;
+  return magicToFormat(MagicStr);
+}
diff --git a/llvm/lib/Remarks/RemarkLinker.cpp b/llvm/lib/Remarks/RemarkLinker.cpp
index b8395aa135d8..0ca6217edfdd 100644
--- a/llvm/lib/Remarks/RemarkLinker.cpp
+++ b/llvm/lib/Remarks/RemarkLinker.cpp
@@ -66,17 +66,10 @@ void RemarkLinker::setExternalFilePrependPath(StringRef PrependPathIn) {
   PrependPath = std::string(PrependPathIn);
 }
 
-Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
-  if (!RemarkFormat) {
-    Expected<Format> ParserFormat = magicToFormat(Buffer);
-    if (!ParserFormat)
-      return ParserFormat.takeError();
-    RemarkFormat = *ParserFormat;
-  }
-
+Error RemarkLinker::link(StringRef Buffer, Format RemarkFormat) {
   Expected<std::unique_ptr<RemarkParser>> MaybeParser =
       createRemarkParserFromMeta(
-          *RemarkFormat, Buffer,
+          RemarkFormat, Buffer,
           PrependPath ? std::optional<StringRef>(StringRef(*PrependPath))
                       : std::optional<StringRef>());
   if (!MaybeParser)
@@ -102,8 +95,7 @@ Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
   return Error::success();
 }
 
-Error RemarkLinker::link(const object::ObjectFile &Obj,
-                         std::optional<Format> RemarkFormat) {
+Error RemarkLinker::link(const object::ObjectFile &Obj, Format RemarkFormat) {
   Expected<std::optional<StringRef>> SectionOrErr =
       getRemarksSectionContents(Obj);
   if (!SectionOrErr)
diff --git a/llvm/lib/Remarks/RemarkParser.cpp b/llvm/lib/Remarks/RemarkParser.cpp
index 5c1690aaa0fe..038fc1d3f485 100644
--- a/llvm/lib/Remarks/RemarkParser.cpp
+++ b/llvm/lib/Remarks/RemarkParser.cpp
@@ -15,6 +15,7 @@
 #include "BitstreamRemarkParser.h"
 #include "YAMLRemarkParser.h"
 #include "llvm-c/Remarks.h"
+#include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Support/CBindingWrapping.h"
 #include <optional>
 
@@ -50,14 +51,18 @@ Expected<StringRef> ParsedStringTable::operator[](size_t Index) const {
 
 Expected<std::unique_ptr<RemarkParser>>
 llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) {
-  switch (ParserFormat) {
+  auto DetectedFormat = detectFormat(ParserFormat, Buf);
+  if (!DetectedFormat)
+    return DetectedFormat.takeError();
+
+  switch (*DetectedFormat) {
   case Format::YAML:
     return std::make_unique<YAMLRemarkParser>(Buf);
   case Format::Bitstream:
     return std::make_unique<BitstreamRemarkParser>(Buf);
   case Format::Unknown:
-    return createStringError(std::make_error_code(std::errc::invalid_argument),
-                             "Unknown remark parser format.");
+  case Format::Auto:
+    break;
   }
   llvm_unreachable("unhandled ParseFormat");
 }
@@ -66,15 +71,19 @@ Expected<std::unique_ptr<RemarkParser>>
 llvm::remarks::createRemarkParserFromMeta(
     Format ParserFormat, StringRef Buf,
     std::optional<StringRef> ExternalFilePrependPath) {
-  switch (ParserFormat) {
+  auto DetectedFormat = detectFormat(ParserFormat, Buf);
+  if (!DetectedFormat)
+    return DetectedFormat.takeError();
+
+  switch (*DetectedFormat) {
   case Format::YAML:
     return createYAMLParserFromMeta(Buf, std::move(ExternalFilePrependPath));
   case Format::Bitstream:
     return createBitstreamParserFromMeta(Buf,
                                          std::move(ExternalFilePrependPath));
   case Format::Unknown:
-    return createStringError(std::make_error_code(std::errc::invalid_argument),
-                             "Unknown remark parser format.");
+  case Format::Auto:
+    break;
   }
   llvm_unreachable("unhandled ParseFormat");
 }
diff --git a/llvm/lib/Remarks/RemarkSerializer.cpp b/llvm/lib/Remarks/RemarkSerializer.cpp
index cc10b91f287a..df1da53d7c8a 100644
--- a/llvm/lib/Remarks/RemarkSerializer.cpp
+++ b/llvm/lib/Remarks/RemarkSerializer.cpp
@@ -22,8 +22,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                                 raw_ostream &OS) {
   switch (RemarksFormat) {
   case Format::Unknown:
+  case Format::Auto:
     return createStringError(std::errc::invalid_argument,
-                             "Unknown remark serializer format.");
+                             "Invalid remark serializer format.");
   case Format::YAML:
     return std::make_unique<YAMLRemarkSerializer>(OS, Mode);
   case Format::Bitstream:
@@ -37,8 +38,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
                                 raw_ostream &OS, remarks::StringTable StrTab) {
   switch (RemarksFormat) {
   case Format::Unknown:
+  case Format::Auto:
     return createStringError(std::errc::invalid_argument,
-                             "Unknown remark serializer format.");
+                             "Invalid remark serializer format.");
   case Format::YAML:
     return std::make_unique<YAMLRemarkSerializer>(OS, Mode, std::move(StrTab));
   case Format::Bitstream:
diff --git a/llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream b/llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream
new file mode 100644
index 000000000000..97b5955f788b
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream
@@ -0,0 +1 @@
+12345678
diff --git a/llvm/test/tools/llvm-remarkutil/annotation-count.test b/llvm/test/tools/llvm-remarkutil/annotation-count.test
index e006220c64f3..ee44ed2035c8 100644
--- a/llvm/test/tools/llvm-remarkutil/annotation-count.test
+++ b/llvm/test/tools/llvm-remarkutil/annotation-count.test
@@ -1,5 +1,7 @@
 RUN: llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
+RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark | FileCheck %s
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --annotation-type=remark | FileCheck %s
 RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="AnnotationSummary" %p/Inputs/annotation-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="AnnotationSummary" | FileCheck %s --check-prefix=COUNT-CHECK
 
diff --git a/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test b/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test
new file mode 100644
index 000000000000..f469eadc07f9
--- /dev/null
+++ b/llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test
@@ -0,0 +1,6 @@
+RUN: not llvm-remarkutil instruction-count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
+RUN: not llvm-remarkutil instruction-mix %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
+RUN: not llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
+RUN: not llvm-remarkutil count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
+
+CHECK: error: Automatic detection of remark format failed. Unknown magic number: '1234'
diff --git a/llvm/test/tools/llvm-remarkutil/empty-file.test b/llvm/test/tools/llvm-remarkutil/empty-file.test
index bdc5fcf87f7b..d9820a088ea8 100644
--- a/llvm/test/tools/llvm-remarkutil/empty-file.test
+++ b/llvm/test/tools/llvm-remarkutil/empty-file.test
@@ -8,6 +8,11 @@ RUN: llvm-remarkutil instruction-count --parser=bitstream %p/Inputs/empty-file -
 RUN: llvm-remarkutil instruction-mix --parser=bitstream %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
 RUN: llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
 RUN: llvm-remarkutil count --parser=bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
+; Parser format auto-detection should treat empty files as bitstream files
+RUN: llvm-remarkutil instruction-count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=SIZEBITSTREAM
+RUN: llvm-remarkutil instruction-mix %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
+RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
+RUN: llvm-remarkutil count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
 
 ; YAMLPARSER: error: document root is not of mapping type.
 
diff --git a/llvm/test/tools/llvm-remarkutil/instruction-count.test b/llvm/test/tools/llvm-remarkutil/instruction-count.test
index d94f4f94cc1d..a0aa6dc98c44 100644
--- a/llvm/test/tools/llvm-remarkutil/instruction-count.test
+++ b/llvm/test/tools/llvm-remarkutil/instruction-count.test
@@ -1,5 +1,7 @@
 RUN: llvm-remarkutil instruction-count --parser=yaml %p/Inputs/instruction-count.yaml | FileCheck %s
+RUN: llvm-remarkutil instruction-count %p/Inputs/instruction-count.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil instruction-count --parser=bitstream | FileCheck %s
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil instruction-count | FileCheck %s
 RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="InstructionCount" %p/Inputs/instruction-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml  | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="InstructionCount" | FileCheck %s --check-prefix=COUNT-CHECK
 RUN: not llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --rremark-name=* %p/Inputs/instruction-count.yaml 2>&1 | FileCheck %s --check-prefix=ERROR-REPOPERATOR -DARG=rremark-name
@@ -18,4 +20,4 @@ RUN: not llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function
 ; COUNT-CHECK: func3,3
 
 ; ERROR-REPOPERATOR: error: invalid argument '--[[ARG]]=*': repetition-operator operand invalid
-; ERROR-BOTHFILTERS: error: conflicting arguments: --remark-name and --rremark-name
\ No newline at end of file
+; ERROR-BOTHFILTERS: error: conflicting arguments: --remark-name and --rremark-name
diff --git a/llvm/test/tools/llvm-remarkutil/instruction-mix.test b/llvm/test/tools/llvm-remarkutil/instruction-mix.test
index 178c1311b2fe..15994679f5d4 100644
--- a/llvm/test/tools/llvm-remarkutil/instruction-mix.test
+++ b/llvm/test/tools/llvm-remarkutil/instruction-mix.test
@@ -1,5 +1,7 @@
 RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml | FileCheck %s
+RUN: llvm-remarkutil instruction-mix %p/Inputs/instruction-mix.yaml | FileCheck %s
 RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-mix.yaml | llvm-remarkutil instruction-mix --parser=bitstream | FileCheck %s
+RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-mix.yaml | llvm-remarkutil instruction-mix | FileCheck %s
 RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --report_style=human | FileCheck %s
 RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --report_style=csv | FileCheck %s --check-prefix=CSV
 RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --rfilter=meow | FileCheck %s --check-prefix=MEOW-RE
@@ -34,4 +36,4 @@ RUN: not llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix
 ; NONE-EXACT: ----------- -----
 ; NONE-NOT: {{.*}}
 
-; ERROR: error: invalid argument '--rfilter=*': repetition-operator operand invalid
\ No newline at end of file
+; ERROR: error: invalid argument '--rfilter=*': repetition-operator operand invalid
diff --git a/llvm/test/tools/llvm-remarkutil/size-diff/no-difference.test b/llvm/test/tools/llvm-remarkutil/size-diff/no-difference.test
index a9b6ba4ae256..8550339bebc4 100644
--- a/llvm/test/tools/llvm-remarkutil/size-diff/no-difference.test
+++ b/llvm/test/tools/llvm-remarkutil/size-diff/no-difference.test
@@ -1,4 +1,7 @@
 RUN: llvm-remarkutil size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml | FileCheck -strict-whitespace %s
+RUN: llvm-remarkutil size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml | FileCheck -strict-whitespace %s
+RUN: llvm-remarkutil yaml2bitstream -o %t.bitstream %p/Inputs/1-func-1-instr-1-stack.yaml
+RUN: llvm-remarkutil size-diff %t.bitstream %p/Inputs/1-func-1-instr-1-stack.yaml | FileCheck -strict-whitespace %s
 
 ; Same file passed twice -> no changes reported.
 
diff --git a/llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h b/llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h
index eb393bc3e304..894ac8354e18 100644
--- a/llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h
+++ b/llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h
@@ -35,9 +35,12 @@
 // Keep Input format and names consistent accross the modes via a macro.
 #define INPUT_FORMAT_COMMAND_LINE_OPTIONS(SUBOPT)                              \
   static cl::opt<Format> InputFormat(                                          \
-      "parser", cl::desc("Input remark format to parse"),                      \
-      cl::values(clEnumValN(Format::YAML, "yaml", "YAML"),                     \
-                 clEnumValN(Format::Bitstream, "bitstream", "Bitstream")),     \
+      "parser", cl::init(Format::Auto),                                        \
+      cl::desc("Input remark format to parse"),                                \
+      cl::values(                                                              \
+          clEnumValN(Format::Auto, "auto", "Automatic detection (default)"),   \
+          clEnumValN(Format::YAML, "yaml", "YAML"),                            \
+          clEnumValN(Format::Bitstream, "bitstream", "Bitstream")),            \
       cl::sub(SUBOPT));
 
 #define DEBUG_LOC_INFO_COMMAND_LINE_OPTIONS(SUBOPT)                            \
diff --git a/llvm/unittests/Remarks/RemarksLinkingTest.cpp b/llvm/unittests/Remarks/RemarksLinkingTest.cpp
index dcd598aaeb5c..89de9e8f4f95 100644
--- a/llvm/unittests/Remarks/RemarksLinkingTest.cpp
+++ b/llvm/unittests/Remarks/RemarksLinkingTest.cpp
@@ -243,10 +243,8 @@ TEST(Remarks, LinkingError) {
     // Check that the prepend path is propagated and fails with the full path.
     // Also ensures that the remark format is correctly auto-detected.
     RL.setExternalFilePrependPath("/baddir/");
-    Error E = RL.link(
-        StringRef("REMARKS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0badfile.opt.yaml",
-                  40),
-        /*RemarkFormat=*/std::nullopt);
+    Error E = RL.link(StringRef(
+        "REMARKS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0badfile.opt.yaml", 40));
     EXPECT_TRUE(static_cast<bool>(E));
     std::string ErrorMessage = toString(std::move(E));
     EXPECT_EQ(StringRef(ErrorMessage).lower(),
-- 
cgit v1.2.3


From 22a69a266d8206b1585dd82d466cd96d01725a65 Mon Sep 17 00:00:00 2001
From: Justin King <jcking@google.com>
Date: Wed, 18 Jun 2025 12:57:49 -0700
Subject: lsan: Support free_sized and free_aligned_sized from C23 (#144604)

Adds support to LSan for `free_sized` and `free_aligned_sized` from C23.

Other sanitizers will be handled with their own separate PRs.

For https://github.com/llvm/llvm-project/issues/144435

This is attempt number 2.

Signed-off-by: Justin King <jcking@google.com>
---
 compiler-rt/lib/lsan/lsan_allocator.cpp            |  4 +++
 compiler-rt/lib/lsan/lsan_allocator.h              |  2 ++
 compiler-rt/lib/lsan/lsan_interceptors.cpp         | 31 ++++++++++++++++++++++
 compiler-rt/lib/lsan/lsan_malloc_mac.cpp           | 23 +++++++++-------
 .../lib/sanitizer_common/sanitizer_malloc_mac.inc  | 16 +++++++++++
 .../sanitizer_platform_interceptors.h              | 11 ++++++++
 .../TestCases/Linux/free_aligned_sized.c           |  2 +-
 .../sanitizer_common/TestCases/Linux/free_sized.c  |  2 +-
 8 files changed, 79 insertions(+), 12 deletions(-)

diff --git a/compiler-rt/lib/lsan/lsan_allocator.cpp b/compiler-rt/lib/lsan/lsan_allocator.cpp
index 493bf5f9efc5..a436d9c07ac6 100644
--- a/compiler-rt/lib/lsan/lsan_allocator.cpp
+++ b/compiler-rt/lib/lsan/lsan_allocator.cpp
@@ -220,6 +220,10 @@ void lsan_free(void *p) {
   Deallocate(p);
 }
 
+void lsan_free_sized(void *p, uptr) { Deallocate(p); }
+
+void lsan_free_aligned_sized(void *p, uptr, uptr) { Deallocate(p); }
+
 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
   return SetErrnoOnNull(Reallocate(stack, p, size, 1));
 }
diff --git a/compiler-rt/lib/lsan/lsan_allocator.h b/compiler-rt/lib/lsan/lsan_allocator.h
index 5eed0cbdb309..2342f11fb5d0 100644
--- a/compiler-rt/lib/lsan/lsan_allocator.h
+++ b/compiler-rt/lib/lsan/lsan_allocator.h
@@ -127,6 +127,8 @@ void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack);
 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack);
 void *lsan_malloc(uptr size, const StackTrace &stack);
 void lsan_free(void *p);
+void lsan_free_sized(void *p, uptr size);
+void lsan_free_aligned_sized(void *p, uptr alignment, uptr size);
 void *lsan_realloc(void *p, uptr size, const StackTrace &stack);
 void *lsan_reallocarray(void *p, uptr nmemb, uptr size,
                         const StackTrace &stack);
diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index a8252cddacf2..6da9d0d9d24f 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -84,6 +84,35 @@ INTERCEPTOR(void, free, void *p) {
   lsan_free(p);
 }
 
+#  if SANITIZER_INTERCEPT_FREE_SIZED
+INTERCEPTOR(void, free_sized, void *p, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  ENSURE_LSAN_INITED;
+  lsan_free_sized(p, size);
+}
+#    define LSAN_MAYBE_INTERCEPT_FREE_SIZED INTERCEPT_FUNCTION(free_sized)
+#  else
+#    define LSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  endif
+
+#  if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+INTERCEPTOR(void, free_aligned_sized, void *p, uptr alignment, uptr size) {
+  if (UNLIKELY(!p))
+    return;
+  if (DlsymAlloc::PointerIsMine(p))
+    return DlsymAlloc::Free(p);
+  ENSURE_LSAN_INITED;
+  lsan_free_aligned_sized(p, alignment, size);
+}
+#    define LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED \
+      INTERCEPT_FUNCTION(free_aligned_sized)
+#  else
+#    define LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
+#  endif
+
 INTERCEPTOR(void*, calloc, uptr nmemb, uptr size) {
   if (DlsymAlloc::Use())
     return DlsymAlloc::Callocate(nmemb, size);
@@ -547,6 +576,8 @@ void InitializeInterceptors() {
 
   INTERCEPT_FUNCTION(malloc);
   INTERCEPT_FUNCTION(free);
+  LSAN_MAYBE_INTERCEPT_FREE_SIZED;
+  LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED;
   LSAN_MAYBE_INTERCEPT_CFREE;
   INTERCEPT_FUNCTION(calloc);
   INTERCEPT_FUNCTION(realloc);
diff --git a/compiler-rt/lib/lsan/lsan_malloc_mac.cpp b/compiler-rt/lib/lsan/lsan_malloc_mac.cpp
index 525c30272ccc..8a16c053da23 100644
--- a/compiler-rt/lib/lsan/lsan_malloc_mac.cpp
+++ b/compiler-rt/lib/lsan/lsan_malloc_mac.cpp
@@ -44,16 +44,19 @@ using namespace __lsan;
   void *p = lsan_valloc(size, stack)
 #define COMMON_MALLOC_FREE(ptr) \
   lsan_free(ptr)
-#define COMMON_MALLOC_SIZE(ptr) \
-  uptr size = lsan_mz_size(ptr)
-#define COMMON_MALLOC_FILL_STATS(zone, stats)
-#define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name) \
-  (void)zone_name; \
-  Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", ptr);
-#define COMMON_MALLOC_NAMESPACE __lsan
-#define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
-#define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
+#  define COMMON_MALLOC_FREE_SIZED(ptr, size) lsan_free_sized(ptr, size)
+#  define COMMON_MALLOC_FREE_ALIGNED_SIZED(ptr, alignment, size) \
+    lsan_free_aligned_sized(ptr, alignment, size)
+#  define COMMON_MALLOC_SIZE(ptr) uptr size = lsan_mz_size(ptr)
+#  define COMMON_MALLOC_FILL_STATS(zone, stats)
+#  define COMMON_MALLOC_REPORT_UNKNOWN_REALLOC(ptr, zone_ptr, zone_name)    \
+    (void)zone_name;                                                        \
+    Report("mz_realloc(%p) -- attempting to realloc unallocated memory.\n", \
+           ptr);
+#  define COMMON_MALLOC_NAMESPACE __lsan
+#  define COMMON_MALLOC_HAS_ZONE_ENUMERATOR 0
+#  define COMMON_MALLOC_HAS_EXTRA_INTROSPECTION_INIT 0
 
-#include "sanitizer_common/sanitizer_malloc_mac.inc"
+#  include "sanitizer_common/sanitizer_malloc_mac.inc"
 
 #endif // SANITIZER_APPLE
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
index 6343eb284afb..be27584f2053 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_malloc_mac.inc
@@ -144,6 +144,22 @@ INTERCEPTOR(void, free, void *ptr) {
   COMMON_MALLOC_FREE(ptr);
 }
 
+#if SANITIZER_INTERCEPT_FREE_SIZED && defined(COMMON_MALLOC_FREE_SIZED)
+INTERCEPTOR(void, free_sized, void *ptr, size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_FREE_SIZED(ptr, size);
+}
+#endif
+
+#if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED && \
+    defined(COMMON_MALLOC_FREE_ALIGNED_SIZED)
+INTERCEPTOR(void, free_aligned_sized, void *ptr, size_t alignment,
+            size_t size) {
+  COMMON_MALLOC_ENTER();
+  COMMON_MALLOC_FREE_ALIGNED_SIZED(ptr, alignment, size);
+}
+#endif
+
 INTERCEPTOR(void *, realloc, void *ptr, size_t size) {
   COMMON_MALLOC_ENTER();
   COMMON_MALLOC_REALLOC(ptr, size);
diff --git a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
index ccc808b60ca7..29987decdff4 100644
--- a/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
+++ b/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h
@@ -663,6 +663,17 @@ SANITIZER_WEAK_IMPORT void *aligned_alloc(__sanitizer::usize __alignment,
 #define SANITIZER_INTERCEPT_GETSERVBYNAME_R SI_GLIBC
 #define SANITIZER_INTERCEPT_GETSERVBYPORT_R SI_GLIBC
 
+// Until free_sized and free_aligned_sized are more generally available,
+// we can only unconditionally intercept on ELF-based platforms where it
+// is okay to have undefined weak symbols.
+#ifdef __ELF__
+#  define SANITIZER_INTERCEPT_FREE_SIZED 1
+#  define SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED 1
+#else
+#  define SANITIZER_INTERCEPT_FREE_SIZED 0
+#  define SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED 0
+#endif
+
 // This macro gives a way for downstream users to override the above
 // interceptor macros irrespective of the platform they are on. They have
 // to do two things:
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
index f4c6c0f973bd..e9cb6f20c5ea 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, lsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
index 0ee2289684d0..8cdf3216e528 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, lsan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
-- 
cgit v1.2.3


From 23b8f11b27f1345cfdd9d03c9ebaccaf81897764 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Wed, 18 Jun 2025 20:59:15 +0100
Subject: [VPlan] Remove redundant VPWidenRecipe constructors (NFC)

Since the removal of VPWidenEVLRecipe, the constructors taking a
VPDefOpcode are not needed any more. Remove them.
---
 llvm/lib/Transforms/Vectorize/VPlan.h             | 21 ++++++---------------
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp |  5 ++---
 2 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index f3306ad7cb8e..ab5ff82a7720 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -1304,24 +1304,15 @@ protected:
 class VPWidenRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
   unsigned Opcode;
 
-protected:
-  VPWidenRecipe(unsigned VPDefOpcode, Instruction &I,
-                ArrayRef<VPValue *> Operands)
-      : VPRecipeWithIRFlags(VPDefOpcode, Operands, I), VPIRMetadata(I),
-        Opcode(I.getOpcode()) {}
-
-  VPWidenRecipe(unsigned VPDefOpcode, unsigned Opcode,
-                ArrayRef<VPValue *> Operands, bool NUW, bool NSW, DebugLoc DL)
-      : VPRecipeWithIRFlags(VPDefOpcode, Operands, WrapFlagsTy(NUW, NSW), DL),
+public:
+  VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands,
+                const VPIRFlags &Flags, DebugLoc DL)
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
         Opcode(Opcode) {}
 
-public:
   VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands)
-      : VPWidenRecipe(VPDef::VPWidenSC, I, Operands) {}
-
-  VPWidenRecipe(unsigned Opcode, ArrayRef<VPValue *> Operands, bool NUW,
-                bool NSW, DebugLoc DL)
-      : VPWidenRecipe(VPDef::VPWidenSC, Opcode, Operands, NUW, NSW, DL) {}
+      : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I),
+        Opcode(I.getOpcode()) {}
 
   ~VPWidenRecipe() override = default;
 
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 11f0f2a93032..cc73ae44f9c0 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2703,9 +2703,8 @@ expandVPMulAccumulateReduction(VPMulAccumulateReductionRecipe *MulAcc) {
   }
 
   std::array<VPValue *, 2> MulOps = {Op0, Op1};
-  auto *Mul = new VPWidenRecipe(
-      Instruction::Mul, ArrayRef(MulOps), MulAcc->hasNoUnsignedWrap(),
-      MulAcc->hasNoSignedWrap(), MulAcc->getDebugLoc());
+  auto *Mul = new VPWidenRecipe(Instruction::Mul, ArrayRef(MulOps), *MulAcc,
+                                MulAcc->getDebugLoc());
   Mul->insertBefore(MulAcc);
 
   auto *Red = new VPReductionRecipe(
-- 
cgit v1.2.3


From a630ca6f6c4727d852d60076d1179c3e9830ca2f Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Wed, 18 Jun 2025 13:06:20 -0700
Subject: [lldb][breakpoint] Grey out disabled breakpoints (#91404)

This commit adds colour settings to the list of breakpoints in order to
grey out breakpoints that have been disabled.
---
 lldb/include/lldb/Core/Debugger.h                 |  4 ++++
 lldb/source/Breakpoint/Breakpoint.cpp             | 15 ++++++++++++++
 lldb/source/Core/CoreProperties.td                | 16 +++++++++++++++
 lldb/source/Core/Debugger.cpp                     | 12 +++++++++++
 lldb/test/API/terminal/TestDisabledBreakpoints.py | 25 +++++++++++++++++++++++
 5 files changed, 72 insertions(+)
 create mode 100644 lldb/test/API/terminal/TestDisabledBreakpoints.py

diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index d73aba1e3ce5..2087ef2a1156 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -307,6 +307,10 @@ public:
 
   llvm::StringRef GetShowProgressAnsiSuffix() const;
 
+  llvm::StringRef GetDisabledAnsiPrefix() const;
+
+  llvm::StringRef GetDisabledAnsiSuffix() const;
+
   bool GetUseAutosuggestion() const;
 
   llvm::StringRef GetAutosuggestionAnsiPrefix() const;
diff --git a/lldb/source/Breakpoint/Breakpoint.cpp b/lldb/source/Breakpoint/Breakpoint.cpp
index 337c1a4ac401..2ed0c9314e3e 100644
--- a/lldb/source/Breakpoint/Breakpoint.cpp
+++ b/lldb/source/Breakpoint/Breakpoint.cpp
@@ -15,6 +15,7 @@
 #include "lldb/Breakpoint/BreakpointResolver.h"
 #include "lldb/Breakpoint/BreakpointResolverFileLine.h"
 #include "lldb/Core/Address.h"
+#include "lldb/Core/Debugger.h"
 #include "lldb/Core/Module.h"
 #include "lldb/Core/ModuleList.h"
 #include "lldb/Core/SearchFilter.h"
@@ -26,6 +27,7 @@
 #include "lldb/Target/SectionLoadList.h"
 #include "lldb/Target/Target.h"
 #include "lldb/Target/ThreadSpec.h"
+#include "lldb/Utility/AnsiTerminal.h"
 #include "lldb/Utility/LLDBLog.h"
 #include "lldb/Utility/Log.h"
 #include "lldb/Utility/Stream.h"
@@ -838,6 +840,13 @@ void Breakpoint::GetDescription(Stream *s, lldb::DescriptionLevel level,
                                 bool show_locations) {
   assert(s != nullptr);
 
+  const bool dim_breakpoint_description =
+      !IsEnabled() && s->AsRawOstream().colors_enabled();
+  if (dim_breakpoint_description)
+    s->Printf("%s", ansi::FormatAnsiTerminalCodes(
+                        GetTarget().GetDebugger().GetDisabledAnsiPrefix())
+                        .c_str());
+
   if (!m_kind_description.empty()) {
     if (level == eDescriptionLevelBrief) {
       s->PutCString(GetBreakpointKind());
@@ -934,6 +943,12 @@ void Breakpoint::GetDescription(Stream *s, lldb::DescriptionLevel level,
     }
     s->IndentLess();
   }
+
+  // Reset the colors back to normal if they were previously greyed out.
+  if (dim_breakpoint_description)
+    s->Printf("%s", ansi::FormatAnsiTerminalCodes(
+                        GetTarget().GetDebugger().GetDisabledAnsiSuffix())
+                        .c_str());
 }
 
 void Breakpoint::GetResolverDescription(Stream *s) {
diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td
index 4d1ea5dfec2e..53dd333f045c 100644
--- a/lldb/source/Core/CoreProperties.td
+++ b/lldb/source/Core/CoreProperties.td
@@ -191,6 +191,22 @@ let Definition = "debugger" in {
             "${separator}${thread.stop-reason}}{ "
             "${separator}{${progress.count} }${progress.message}}">,
         Desc<"The default statusline format string.">;
+
+  def ShowDisabledAnsiPrefix
+      : Property<"disable-ansi-prefix", "String">,
+        Global,
+        DefaultStringValue<"${ansi.faint}">,
+        Desc<"If something has been disabled in a color-enabled terminal, use "
+             "the ANSI terminal code specified immediately before whatever has "
+             "been disabled.">;
+  def ShowDisabledAnsiSuffix
+      : Property<"disable-ansi-suffix", "String">,
+        Global,
+        DefaultStringValue<"${ansi.normal}">,
+        Desc<"When somehing has been disabled in a color-enabled terminal, use "
+             "the ANSI terminal code specified immediately after whatever has "
+             "been disabled.">;
+
   def UseSourceCache: Property<"use-source-cache", "Boolean">,
     Global,
     DefaultTrue,
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index 81037d3def81..c9935f2d745f 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -510,6 +510,18 @@ llvm::StringRef Debugger::GetSeparator() const {
       idx, g_debugger_properties[idx].default_cstr_value);
 }
 
+llvm::StringRef Debugger::GetDisabledAnsiPrefix() const {
+  const uint32_t idx = ePropertyShowDisabledAnsiPrefix;
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
+}
+
+llvm::StringRef Debugger::GetDisabledAnsiSuffix() const {
+  const uint32_t idx = ePropertyShowDisabledAnsiSuffix;
+  return GetPropertyAtIndexAs<llvm::StringRef>(
+      idx, g_debugger_properties[idx].default_cstr_value);
+}
+
 bool Debugger::SetSeparator(llvm::StringRef s) {
   constexpr uint32_t idx = ePropertySeparator;
   bool ret = SetPropertyAtIndex(idx, s);
diff --git a/lldb/test/API/terminal/TestDisabledBreakpoints.py b/lldb/test/API/terminal/TestDisabledBreakpoints.py
new file mode 100644
index 000000000000..a644c94c8a17
--- /dev/null
+++ b/lldb/test/API/terminal/TestDisabledBreakpoints.py
@@ -0,0 +1,25 @@
+"""
+Test that disabling breakpoints and viewing them in a list uses the correct ANSI color settings when colors are enabled and disabled.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+from lldbsuite.test.lldbpexpect import PExpectTest
+
+import io
+
+
+class DisabledBreakpointsTest(PExpectTest):
+    @add_test_categories(["pexpect"])
+    def test_disabling_breakpoints_with_color(self):
+        """Test that disabling a breakpoint and viewing the breakpoints list uses the specified ANSI color prefix."""
+        ansi_red_color_code = "\x1b[31m"
+
+        self.launch(use_colors=True, dimensions=(100, 100))
+        self.expect('settings set disable-ansi-prefix "${ansi.fg.red}"')
+        self.expect("b main")
+        self.expect("br dis")
+        self.expect("br l", substrs=[ansi_red_color_code + "1:"])
+        self.quit()
-- 
cgit v1.2.3


From a88e655809655eec8fa85366318fb3c4a0baa113 Mon Sep 17 00:00:00 2001
From: Andrew Rogers <andrurogerz@gmail.com>
Date: Wed, 18 Jun 2025 13:08:05 -0700
Subject: [llvm] build Blake3 source with LLVM_EXPORTS defined (#144753)

## Purpose
This patch ensures that the BLAKE3 implementation in the LLVM Support
library exports its public interface with `__declspec(dllexport)` when
building LLVM as a Windows DLL.

## Background
The effort to support building LLVM as a Windows DLL is tracked in
#109483. Additional context is provided in [this
discourse](https://discourse.llvm.org/t/psa-annotating-llvm-public-interface/85307).

## Overview
Replicate [this
logic](https://github.com/llvm/llvm-project/blob/main/llvm/cmake/modules/AddLLVM.cmake#L662-L664)
from `llvm_add_library()` for the `LLVMSupportBlake3` target. Without
this change, the `llvm_blake_` functions will only be annotated with
`__declspec(dllimport)` when building LLVM as a Windows DLL which leads
to inconsistent DLL linkage warnings from MSVC and `clang-cl`.
---
 llvm/lib/Support/BLAKE3/CMakeLists.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/llvm/lib/Support/BLAKE3/CMakeLists.txt b/llvm/lib/Support/BLAKE3/CMakeLists.txt
index 99cb78881ec4..eae2b0280e5d 100644
--- a/llvm/lib/Support/BLAKE3/CMakeLists.txt
+++ b/llvm/lib/Support/BLAKE3/CMakeLists.txt
@@ -85,3 +85,9 @@ endif()
 add_library(LLVMSupportBlake3 OBJECT EXCLUDE_FROM_ALL ${LLVM_BLAKE3_FILES})
 set_target_properties(LLVMSupportBlake3 PROPERTIES FOLDER "LLVM/Libraries")
 llvm_update_compile_flags(LLVMSupportBlake3)
+if(LLVM_BUILD_LLVM_DYLIB OR BUILD_SHARED_LIBS)
+  # Since LLVMSupportBlake3 is not defined using llvm_add_library(), we must
+  # define LLVM_EXPORTS here so its public interface is annotated with
+  # __declspec(dllexport) when building as a DLL on Windows.
+  target_compile_definitions(LLVMSupportBlake3 PRIVATE LLVM_EXPORTS)
+endif()
-- 
cgit v1.2.3


From 96bbe472ef01e5f34bfeabedceea397889ff1119 Mon Sep 17 00:00:00 2001
From: Jakub Kuderski <jakub@nod-labs.com>
Date: Wed, 18 Jun 2025 16:15:06 -0400
Subject: Revert "[mlir][spirv] Fix int type declaration duplication when
 serializing" and follow up commits (#144773)

This reverts the following PRs:
* https://github.com/llvm/llvm-project/pull/143108
* https://github.com/llvm/llvm-project/pull/144538
* https://github.com/llvm/llvm-project/pull/144685

Reverting because this disabled tests when building without the llvm
spirv backend enabled.
---
 mlir/lib/Target/SPIRV/Serialization/Serializer.cpp     | 13 -------------
 mlir/test/CMakeLists.txt                               |  6 ------
 mlir/test/Target/SPIRV/constant.mlir                   |  5 +----
 mlir/test/Target/SPIRV/lit.local.cfg                   |  7 -------
 mlir/test/lit.cfg.py                                   |  1 -
 mlir/test/lit.site.cfg.py.in                           |  4 +---
 utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel |  1 -
 7 files changed, 2 insertions(+), 35 deletions(-)
 delete mode 100644 mlir/test/Target/SPIRV/lit.local.cfg

diff --git a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
index 56c64f38fe29..d258bfd85296 100644
--- a/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
+++ b/mlir/lib/Target/SPIRV/Serialization/Serializer.cpp
@@ -446,19 +446,6 @@ LogicalResult Serializer::processType(Location loc, Type type,
 LogicalResult
 Serializer::processTypeImpl(Location loc, Type type, uint32_t &typeID,
                             SetVector<StringRef> &serializationCtx) {
-
-  // Map unsigned integer types to singless integer types.
-  // This is needed otherwise the generated spirv assembly will contain
-  // twice a type declaration (like OpTypeInt 32 0) which is no permitted and
-  // such module fails validation. Indeed at MLIR level the two types are
-  // different and lookup in the cache below misses.
-  // Note: This conversion needs to happen here before the type is looked up in
-  // the cache.
-  if (type.isUnsignedInteger()) {
-    type = IntegerType::get(loc->getContext(), type.getIntOrFloatBitWidth(),
-                            IntegerType::SignednessSemantics::Signless);
-  }
-
   typeID = getTypeID(type);
   if (typeID)
     return success();
diff --git a/mlir/test/CMakeLists.txt b/mlir/test/CMakeLists.txt
index 89568e7766ae..ac8b44f53aeb 100644
--- a/mlir/test/CMakeLists.txt
+++ b/mlir/test/CMakeLists.txt
@@ -68,7 +68,6 @@ endif()
 llvm_canonicalize_cmake_booleans(
   LLVM_BUILD_EXAMPLES
   LLVM_HAS_NVPTX_TARGET
-  LLVM_INCLUDE_SPIRV_TOOLS_TESTS
   MLIR_ENABLE_BINDINGS_PYTHON
   MLIR_ENABLE_CUDA_RUNNER
   MLIR_ENABLE_ROCM_CONVERSIONS
@@ -218,11 +217,6 @@ if(MLIR_ENABLE_BINDINGS_PYTHON)
   )
 endif()
 
-if (LLVM_INCLUDE_SPIRV_TOOLS_TESTS)
-  list(APPEND MLIR_TEST_DEPENDS spirv-as)
-  list(APPEND MLIR_TEST_DEPENDS spirv-val)
-endif()
-
 # This target can be used to just build the dependencies
 # for the check-mlir target without executing the tests.
 # This is useful for bots when splitting the build step
diff --git a/mlir/test/Target/SPIRV/constant.mlir b/mlir/test/Target/SPIRV/constant.mlir
index 50d9b09ee004..8d4e53418b70 100644
--- a/mlir/test/Target/SPIRV/constant.mlir
+++ b/mlir/test/Target/SPIRV/constant.mlir
@@ -1,7 +1,6 @@
 // RUN: mlir-translate --no-implicit-module --test-spirv-roundtrip %s | FileCheck %s
-// RUN: %if spirv-tools %{ mlir-translate -no-implicit-module -serialize-spirv %s | spirv-val %}
 
-spirv.module Logical Vulkan requires #spirv.vce<v1.3, [VulkanMemoryModel, Shader, Int64, Int16, Int8, Float64, Float16, CooperativeMatrixKHR], [SPV_KHR_vulkan_memory_model, SPV_KHR_cooperative_matrix]> {
+spirv.module Logical GLSL450 requires #spirv.vce<v1.0, [Shader], []> {
   // CHECK-LABEL: @bool_const
   spirv.func @bool_const() -> () "None" {
     // CHECK: spirv.Constant true
@@ -306,6 +305,4 @@ spirv.module Logical Vulkan requires #spirv.vce<v1.3, [VulkanMemoryModel, Shader
     %coop = spirv.Constant dense<4> : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
     spirv.ReturnValue %coop : !spirv.coopmatrix<16x16xi8, Subgroup, MatrixAcc>
   }
-
-  spirv.EntryPoint "GLCompute" @bool_const
 }
diff --git a/mlir/test/Target/SPIRV/lit.local.cfg b/mlir/test/Target/SPIRV/lit.local.cfg
deleted file mode 100644
index 167c454db518..000000000000
--- a/mlir/test/Target/SPIRV/lit.local.cfg
+++ /dev/null
@@ -1,7 +0,0 @@
-if not "SPIRV" in config.root.targets:
-    config.unsupported = True
-
-if config.spirv_tools_tests:
-    config.available_features.add("spirv-tools")
-    config.substitutions.append(("spirv-as", os.path.join(config.llvm_tools_dir, "spirv-as")))
-    config.substitutions.append(("spirv-val", os.path.join(config.llvm_tools_dir, "spirv-val")))
diff --git a/mlir/test/lit.cfg.py b/mlir/test/lit.cfg.py
index a6f1ac0d568f..9b5cadd62bef 100644
--- a/mlir/test/lit.cfg.py
+++ b/mlir/test/lit.cfg.py
@@ -332,7 +332,6 @@ if config.enable_assertions:
 else:
     config.available_features.add("noasserts")
 
-config.targets = frozenset(config.targets_to_build.split())
 
 def have_host_jit_feature_support(feature_name):
     mlir_runner_exe = lit.util.which("mlir-runner", config.mlir_tools_dir)
diff --git a/mlir/test/lit.site.cfg.py.in b/mlir/test/lit.site.cfg.py.in
index 77f24e0f29b0..132aabe13594 100644
--- a/mlir/test/lit.site.cfg.py.in
+++ b/mlir/test/lit.site.cfg.py.in
@@ -5,8 +5,6 @@ import sys
 config.target_triple = "@LLVM_TARGET_TRIPLE@"
 config.llvm_src_root = "@LLVM_SOURCE_DIR@"
 config.llvm_tools_dir = lit_config.substitute("@LLVM_TOOLS_DIR@")
-config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
-config.targets_to_build = "@TARGETS_TO_BUILD@"
 config.llvm_shlib_ext = "@SHLIBEXT@"
 config.llvm_shlib_dir = lit_config.substitute(path(r"@SHLIBDIR@"))
 config.python_executable = "@Python3_EXECUTABLE@"
@@ -43,7 +41,7 @@ config.mlir_run_amx_tests = @MLIR_RUN_AMX_TESTS@
 config.mlir_run_arm_sve_tests = @MLIR_RUN_ARM_SVE_TESTS@
 # This is a workaround for the fact that LIT's:
 #   %if <cond>
-# requires <cond> to be in the set of available features.
+# requires <cond> to be in the set of available features. 
 # TODO: Update LIT's TestRunner so that this is not required.
 if config.mlir_run_arm_sve_tests:
     config.available_features.add("mlir_arm_sve_tests")
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index a439fdd50d21..51731b1e8f74 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -37,7 +37,6 @@ expand_template(
         # All disabled, but required to substituted because they are not in quotes.
         "@LLVM_BUILD_EXAMPLES@": "0",
         "@LLVM_HAS_NVPTX_TARGET@": "0",
-        "@LLVM_INCLUDE_SPIRV_TOOLS_TESTS@": "0",
         "@MLIR_ENABLE_CUDA_RUNNER@": "0",
         "@MLIR_ENABLE_ROCM_CONVERSIONS@": "0",
         "@MLIR_ENABLE_ROCM_RUNNER@": "0",
-- 
cgit v1.2.3


From b85e92990fdec32ec7512dec7bd36d945f8e0144 Mon Sep 17 00:00:00 2001
From: Alexey Karyakin <akaryaki@quicinc.com>
Date: Wed, 18 Jun 2025 15:26:38 -0500
Subject: Hexagon v87 v89 elf flags (#144584)

---
 llvm/include/llvm/BinaryFormat/ELF.h | 64 +++++++++++++++++++++---------------
 1 file changed, 37 insertions(+), 27 deletions(-)

diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 97f417263aff..dcf1f1c6d7b2 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -614,33 +614,7 @@ enum {
 
 // Hexagon-specific e_flags
 enum {
-  // Object processor version flags, bits[11:0]
-  EF_HEXAGON_MACH_V2 = 0x00000001,   // Hexagon V2
-  EF_HEXAGON_MACH_V3 = 0x00000002,   // Hexagon V3
-  EF_HEXAGON_MACH_V4 = 0x00000003,   // Hexagon V4
-  EF_HEXAGON_MACH_V5 = 0x00000004,   // Hexagon V5
-  EF_HEXAGON_MACH_V55 = 0x00000005,  // Hexagon V55
-  EF_HEXAGON_MACH_V60 = 0x00000060,  // Hexagon V60
-  EF_HEXAGON_MACH_V61 = 0x00000061,  // Hexagon V61
-  EF_HEXAGON_MACH_V62 = 0x00000062,  // Hexagon V62
-  EF_HEXAGON_MACH_V65 = 0x00000065,  // Hexagon V65
-  EF_HEXAGON_MACH_V66 = 0x00000066,  // Hexagon V66
-  EF_HEXAGON_MACH_V67 = 0x00000067,  // Hexagon V67
-  EF_HEXAGON_MACH_V67T = 0x00008067, // Hexagon V67T
-  EF_HEXAGON_MACH_V68 = 0x00000068,  // Hexagon V68
-  EF_HEXAGON_MACH_V69 = 0x00000069,  // Hexagon V69
-  EF_HEXAGON_MACH_V71 = 0x00000071,  // Hexagon V71
-  EF_HEXAGON_MACH_V71T = 0x00008071, // Hexagon V71T
-  EF_HEXAGON_MACH_V73 = 0x00000073,  // Hexagon V73
-  EF_HEXAGON_MACH_V75 = 0x00000075,  // Hexagon V75
-  EF_HEXAGON_MACH_V77 = 0x00000077,  // Hexagon V77
-  EF_HEXAGON_MACH_V79 = 0x00000079,  // Hexagon V79
-  EF_HEXAGON_MACH_V81 = 0x00000081,  // Hexagon V81
-  EF_HEXAGON_MACH_V83 = 0x00000083,  // Hexagon V83
-  EF_HEXAGON_MACH_V85 = 0x00000085,  // Hexagon V85
-  EF_HEXAGON_MACH = 0x000003ff,      // Hexagon V..
-
-  // Highest ISA version flags
+  // Hexagon ISA version, bits[11:0]
   EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0]
                                     // of e_flags
   EF_HEXAGON_ISA_V2 = 0x00000010,   // Hexagon V2 ISA
@@ -664,7 +638,43 @@ enum {
   EF_HEXAGON_ISA_V81 = 0x00000081,  // Hexagon V81 ISA
   EF_HEXAGON_ISA_V83 = 0x00000083,  // Hexagon V83 ISA
   EF_HEXAGON_ISA_V85 = 0x00000085,  // Hexagon V85 ISA
+  EF_HEXAGON_ISA_V87 = 0x00000087,  // Hexagon V87 ISA
+  EF_HEXAGON_ISA_V89 = 0x00000089,  // Hexagon V89 ISA
   EF_HEXAGON_ISA = 0x000003ff,      // Hexagon V.. ISA
+
+  // Tiny core flag, bit[15]
+  EF_HEXAGON_TINY_CORE = 0x00008000, // Hexagon Tiny Core
+
+  // Hexagon processor version, bits[15:0]
+  EF_HEXAGON_MACH_V2 = 0x00000001,          // Hexagon V2
+  EF_HEXAGON_MACH_V3 = 0x00000002,          // Hexagon V3
+  EF_HEXAGON_MACH_V4 = 0x00000003,          // Hexagon V4
+  EF_HEXAGON_MACH_V5 = 0x00000004,          // Hexagon V5
+  EF_HEXAGON_MACH_V55 = 0x00000005,         // Hexagon V55
+  EF_HEXAGON_MACH_V60 = EF_HEXAGON_ISA_V60, // Hexagon V60
+  EF_HEXAGON_MACH_V61 = EF_HEXAGON_ISA_V61, // Hexagon V61
+  EF_HEXAGON_MACH_V62 = EF_HEXAGON_ISA_V62, // Hexagon V62
+  EF_HEXAGON_MACH_V65 = EF_HEXAGON_ISA_V65, // Hexagon V65
+  EF_HEXAGON_MACH_V66 = EF_HEXAGON_ISA_V66, // Hexagon V66
+  EF_HEXAGON_MACH_V67 = EF_HEXAGON_ISA_V67, // Hexagon V67
+  EF_HEXAGON_MACH_V67T =
+      EF_HEXAGON_ISA_V67 | EF_HEXAGON_TINY_CORE, // Hexagon V67T
+  EF_HEXAGON_MACH_V68 = EF_HEXAGON_ISA_V68,      // Hexagon V68
+  EF_HEXAGON_MACH_V69 = EF_HEXAGON_ISA_V69,      // Hexagon V69
+  EF_HEXAGON_MACH_V71 = EF_HEXAGON_ISA_V71,      // Hexagon V71
+  EF_HEXAGON_MACH_V71T =
+      EF_HEXAGON_ISA_V71 | EF_HEXAGON_TINY_CORE, // Hexagon V71T
+  EF_HEXAGON_MACH_V73 = EF_HEXAGON_ISA_V73,      // Hexagon V73
+  EF_HEXAGON_MACH_V75 = EF_HEXAGON_ISA_V75,      // Hexagon V75
+  EF_HEXAGON_MACH_V77 = EF_HEXAGON_ISA_V77,      // Hexagon V77
+  EF_HEXAGON_MACH_V79 = EF_HEXAGON_ISA_V79,      // Hexagon V79
+  EF_HEXAGON_MACH_V81 = EF_HEXAGON_ISA_V81,      // Hexagon V81
+  EF_HEXAGON_MACH_V83 = EF_HEXAGON_ISA_V83,      // Hexagon V83
+  EF_HEXAGON_MACH_V85 = EF_HEXAGON_ISA_V85,      // Hexagon V85
+  EF_HEXAGON_MACH_V87 = EF_HEXAGON_ISA_V87,      // Hexagon V87
+  EF_HEXAGON_MACH_V89 = EF_HEXAGON_ISA_V89,      // Hexagon V89
+
+  EF_HEXAGON_MACH = 0x0000ffff, // Hexagon V..
 };
 
 // Hexagon-specific section indexes for common small data
-- 
cgit v1.2.3


From 7aecd7ecacb4b305b94149f3cfcef306a9da6beb Mon Sep 17 00:00:00 2001
From: Diego Caballero <dieg0ca6aller0@gmail.com>
Date: Wed, 18 Jun 2025 13:45:43 -0700
Subject: [mlir][Vector] Add `vector.to_elements` op (#141457)

This PR introduces the `vector.to_elements` op, which decomposes a
vector into its scalar elements. This operation is symmetrical to the
existing `vector.from_elements`.

Examples:

```
    // Decompose a 0-D vector.
    %0 = vector.to_elements %v0 : vector<f32>
    // %0 = %v0[0]

    // Decompose a 1-D vector.
    %0:2 = vector.to_elements %v1 : vector<2xf32>
    // %0#0 = %v1[0]
    // %0#1 = %v1[1]

    // Decompose a 2-D.
    %0:6 = vector.to_elements %v2 : vector<2x3xf32>
    // %0#0 = %v2[0, 0]
    // %0#1 = %v2[0, 1]
    // %0#2 = %v2[0, 2]
    // %0#3 = %v2[1, 0]
    // %0#4 = %v2[1, 1]
    // %0#5 = %v2[1, 2]
```

This op is aimed at reducing code size when modeling "structured" vector
extractions and simplifying canonicalizations of large sequences of
`vector.extract` and `vector.insert` ops into `vector.shuffle` and other
sophisticated ops that can re-arrange vector elements.
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td | 85 +++++++++++++++++++-----
 mlir/include/mlir/IR/OpBase.td                   | 19 ++++++
 mlir/lib/TableGen/Operator.cpp                   | 31 +++++++++
 mlir/test/Dialect/Vector/invalid.mlir            | 24 +++++--
 mlir/test/Dialect/Vector/ops.mlir                | 18 +++++
 mlir/tools/mlir-tblgen/OpFormatGen.cpp           | 26 ++++++++
 6 files changed, 181 insertions(+), 22 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 8353314ed958..125cd4645ccc 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -790,40 +790,89 @@ def Vector_FMAOp :
   }];
 }
 
+def Vector_ToElementsOp : Vector_Op<"to_elements", [
+    Pure,
+    ShapedTypeMatchesElementCountAndTypes<"source", "elements">]> {
+  let summary = "operation that decomposes a vector into all its scalar elements";
+  let description = [{
+    This operation decomposes all the scalar elements from a vector. The
+    decomposed scalar elements are returned in row-major order. The number of
+    scalar results must match the number of elements in the input vector type.
+    All the result elements have the same result type, which must match the
+    element type of the input vector. Scalable vectors are not supported.
+
+    Examples:
+
+    ```mlir
+    // Decompose a 0-D vector.
+    %0 = vector.to_elements %v0 : vector<f32>
+    // %0 = %v0[0]
+
+    // Decompose a 1-D vector.
+    %0:2 = vector.to_elements %v1 : vector<2xf32>
+    // %0#0 = %v1[0]
+    // %0#1 = %v1[1]
+
+    // Decompose a 2-D.
+    %0:6 = vector.to_elements %v2 : vector<2x3xf32>
+    // %0#0 = %v2[0, 0]
+    // %0#1 = %v2[0, 1]
+    // %0#2 = %v2[0, 2]
+    // %0#3 = %v2[1, 0]
+    // %0#4 = %v2[1, 1]
+    // %0#5 = %v2[1, 2]
+
+    // Decompose a 3-D vector.
+    %0:6 = vector.to_elements %v3 : vector<3x1x2xf32>
+    // %0#0 = %v3[0, 0, 0]
+    // %0#1 = %v3[0, 0, 1]
+    // %0#2 = %v3[1, 0, 0]
+    // %0#3 = %v3[1, 0, 1]
+    // %0#4 = %v3[2, 0, 0]
+    // %0#5 = %v3[2, 0, 1]
+    ```
+  }];
+
+  let arguments = (ins AnyVectorOfAnyRank:$source);
+  let results = (outs Variadic<AnyType>:$elements);
+  let assemblyFormat = "$source attr-dict `:` type($source)";
+}
+
 def Vector_FromElementsOp : Vector_Op<"from_elements", [
     Pure,
-    TypesMatchWith<"operand types match result element type",
-                   "result", "elements", "SmallVector<Type>("
-                   "::llvm::cast<VectorType>($_self).getNumElements(), "
-                   "::llvm::cast<VectorType>($_self).getElementType())">]> {
+    ShapedTypeMatchesElementCountAndTypes<"dest", "elements">]> {
   let summary = "operation that defines a vector from scalar elements";
   let description = [{
     This operation defines a vector from one or multiple scalar elements. The
-    number of elements must match the number of elements in the result type.
-    All elements must have the same type, which must match the element type of
-    the result vector type.
-
-    `elements` are a flattened version of the result vector in row-major order.
+    scalar elements are arranged in row-major within the vector. The number of
+    elements must match the number of elements in the result type. All elements
+    must have the same type, which must match the element type of the result
+    vector type. Scalable vectors are not supported.
 
-    Example:
+    Examples:
 
     ```mlir
-    // %f1
+    // Define a 0-D vector.
     %0 = vector.from_elements %f1 : vector<f32>
-    // [%f1, %f2]
+    // [%f1]
+
+    // Define a 1-D vector.
     %1 = vector.from_elements %f1, %f2 : vector<2xf32>
-    // [[%f1, %f2, %f3], [%f4, %f5, %f6]]
+    // [%f1, %f2]
+
+    // Define a 2-D vector.
     %2 = vector.from_elements %f1, %f2, %f3, %f4, %f5, %f6 : vector<2x3xf32>
-    // [[[%f1, %f2]], [[%f3, %f4]], [[%f5, %f6]]]
+    // [[%f1, %f2, %f3], [%f4, %f5, %f6]]
+
+    // Define a 3-D vector.
     %3 = vector.from_elements %f1, %f2, %f3, %f4, %f5, %f6 : vector<3x1x2xf32>
+    // [[[%f1, %f2]], [[%f3, %f4]], [[%f5, %f6]]]
     ```
-
-    Note, scalable vectors are not supported.
   }];
 
   let arguments = (ins Variadic<AnyType>:$elements);
-  let results = (outs AnyFixedVectorOfAnyRank:$result);
-  let assemblyFormat = "$elements attr-dict `:` type($result)";
+  let results = (outs AnyFixedVectorOfAnyRank:$dest);
+  let assemblyFormat = "$elements attr-dict `:` type($dest)";
   let hasCanonicalizer = 1;
 }
 
diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index 51b60972203e..b3fabe409806 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -556,6 +556,25 @@ class AllShapesMatch<list<string> names> :
 class AllTypesMatch<list<string> names> :
     AllMatchSameOperatorTrait<names, "$_self.getType()", "type">;
 
+// A type constraint that verifies that a shaped type matches the size and
+// element type of a container with element types. More specifically, it denotes
+// shapedArg.getType().getNumElements() == elementsArg.size() &&
+// shapedArg.getType().getElementType() == elementsArg[i].getType(), for i in
+// [0, elementsArg.size()).
+class ShapedTypeMatchesElementCountAndTypes<string shapedArg,
+                                            string elementsArg> :
+    PredOpTrait<"shaped type '" # shapedArg # "' matches '" # elementsArg # "' "
+                "element count and types",
+        And<[CPred<ElementCount<shapedArg>.result # " == "
+                     "$" # elementsArg # ".getTypes().size()">,
+             CPred<"::llvm::all_of($" # elementsArg # ".getTypes(), "
+                     "[&](::mlir::Type t) { return t == "
+                       # ElementType<shapedArg>.result # "; })">]>> {
+
+  string shaped = shapedArg;
+  string elements = elementsArg;
+}
+
 // A type constraint that denotes `transform(lhs.getType()) == rhs.getType()`.
 // An optional comparator function may be provided that changes the above form
 // into: `comparator(transform(lhs.getType()), rhs.getType())`.
diff --git a/mlir/lib/TableGen/Operator.cpp b/mlir/lib/TableGen/Operator.cpp
index 2544f0a1b91b..07520a2f94d7 100644
--- a/mlir/lib/TableGen/Operator.cpp
+++ b/mlir/lib/TableGen/Operator.cpp
@@ -468,6 +468,37 @@ void Operator::populateTypeInferenceInfo(
       continue;
     }
 
+    // The `ShapedTypeMatchesElementCountAndTypes` trait represents a 1 -> 1
+    // type inference edge where a shaped type matches element count and types
+    // of variadic elements.
+    if (def.isSubClassOf("ShapedTypeMatchesElementCountAndTypes")) {
+      StringRef shapedArg = def.getValueAsString("shaped");
+      StringRef elementsArg = def.getValueAsString("elements");
+
+      int shapedIndex = argumentsAndResultsIndex.lookup(shapedArg);
+      int elementsIndex = argumentsAndResultsIndex.lookup(elementsArg);
+
+      // Handle result type inference from shaped type to variadic elements.
+      if (InferredResultType::isResultIndex(elementsIndex) &&
+          InferredResultType::isArgIndex(shapedIndex)) {
+        int resultIndex = InferredResultType::unmapResultIndex(elementsIndex);
+        ResultTypeInference &infer = inference[resultIndex];
+        if (!infer.inferred) {
+          infer.sources.emplace_back(
+              shapedIndex,
+              "::llvm::SmallVector<::mlir::Type>(::llvm::cast<::mlir::"
+              "ShapedType>($_self).getNumElements(), "
+              "::llvm::cast<::mlir::ShapedType>($_self).getElementType())");
+          infer.inferred = true;
+        }
+      }
+
+      // Type inference in the opposite direction is not possible as the actual
+      // shaped type can't be inferred from the variadic elements.
+
+      continue;
+    }
+
     if (!def.isSubClassOf("AllTypesMatch"))
       continue;
 
diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir
index 04810ed52584..ec7cee7b2c64 100644
--- a/mlir/test/Dialect/Vector/invalid.mlir
+++ b/mlir/test/Dialect/Vector/invalid.mlir
@@ -1896,7 +1896,24 @@ func.func @deinterleave_scalable_rank_fail(%vec : vector<2x[4]xf32>) {
 
 // -----
 
-func.func @invalid_from_elements(%a: f32) {
+func.func @to_elements_wrong_num_results(%a: vector<1x1x2xf32>) {
+  // expected-error @+1 {{operation defines 2 results but was provided 4 to bind}}
+  %0:4 = vector.to_elements %a : vector<1x1x2xf32>
+  return
+}
+
+// -----
+
+func.func @to_elements_wrong_result_type(%a: vector<2xf32>) -> i32 {
+  // expected-error @+3 {{use of value '%0' expects different type than prior uses: 'i32'}}
+  // expected-note @+1 {{prior use here}}
+  %0:2 = vector.to_elements %a : vector<2xf32>
+  return %0#0 : i32
+}
+
+// -----
+
+func.func @from_elements_wrong_num_operands(%a: f32) {
   // expected-error @+1 {{'vector.from_elements' number of operands and types do not match: got 1 operands and 2 types}}
   vector.from_elements %a : vector<2xf32>
   return
@@ -1905,16 +1922,15 @@ func.func @invalid_from_elements(%a: f32) {
 // -----
 
 // expected-note @+1 {{prior use here}}
-func.func @invalid_from_elements(%a: f32, %b: i32) {
+func.func @from_elements_wrong_operand_type(%a: f32, %b: i32) {
   // expected-error @+1 {{use of value '%b' expects different type than prior uses: 'f32' vs 'i32'}}
   vector.from_elements %a, %b : vector<2xf32>
   return
 }
-
 // -----
 
 func.func @invalid_from_elements_scalable(%a: f32, %b: i32) {
-  // expected-error @+1 {{'result' must be fixed-length vector of any type values, but got 'vector<[2]xf32>'}}
+  // expected-error @+1 {{'dest' must be fixed-length vector of any type values, but got 'vector<[2]xf32>'}}
   vector.from_elements %a, %b : vector<[2]xf32>
   return
 }
diff --git a/mlir/test/Dialect/Vector/ops.mlir b/mlir/test/Dialect/Vector/ops.mlir
index f3220aed4360..c59f7bd00190 100644
--- a/mlir/test/Dialect/Vector/ops.mlir
+++ b/mlir/test/Dialect/Vector/ops.mlir
@@ -1175,6 +1175,24 @@ func.func @deinterleave_nd_scalable(%arg:vector<2x3x4x[6]xf32>) -> (vector<2x3x4
   return %0, %1 : vector<2x3x4x[3]xf32>, vector<2x3x4x[3]xf32>
 }
 
+// CHECK-LABEL: func @to_elements(
+//  CHECK-SAME:     %[[A_VEC:.*]]: vector<f32>, %[[B_VEC:.*]]: vector<1xf32>,
+//  CHECK-SAME:     %[[C_VEC:.*]]: vector<1x2xf32>, %[[D_VEC:.*]]: vector<2x2xf32>)
+func.func @to_elements(%a_vec : vector<f32>, %b_vec : vector<1xf32>,
+                       %c_vec : vector<1x2xf32>, %d_vec : vector<2x2xf32>)
+                   -> (f32, f32, f32, f32, f32, f32, f32, f32) {
+  // CHECK: %[[A_ELEMS:.*]] = vector.to_elements %[[A_VEC]] : vector<f32>
+  %0 = vector.to_elements %a_vec : vector<f32>
+  // CHECK: %[[B_ELEMS:.*]] = vector.to_elements %[[B_VEC]] : vector<1xf32>
+  %1 = vector.to_elements %b_vec : vector<1xf32>
+  // CHECK: %[[C_ELEMS:.*]]:2 = vector.to_elements %[[C_VEC]] : vector<1x2xf32>
+  %2:2 = vector.to_elements %c_vec : vector<1x2xf32>
+  // CHECK: %[[D_ELEMS:.*]]:4 = vector.to_elements %[[D_VEC]] : vector<2x2xf32>
+  %3:4 = vector.to_elements %d_vec : vector<2x2xf32>
+  // CHECK: return %[[A_ELEMS]], %[[B_ELEMS]], %[[C_ELEMS]]#0, %[[C_ELEMS]]#1, %[[D_ELEMS]]#0, %[[D_ELEMS]]#1, %[[D_ELEMS]]#2, %[[D_ELEMS]]#3
+  return %0, %1, %2#0, %2#1, %3#0, %3#1, %3#2, %3#3: f32, f32, f32, f32, f32, f32, f32, f32
+}
+
 // CHECK-LABEL: func @from_elements(
 //  CHECK-SAME:     %[[a:.*]]: f32, %[[b:.*]]: f32)
 func.func @from_elements(%a: f32, %b: f32) -> (vector<f32>, vector<1xf32>, vector<1x2xf32>, vector<2x2xf32>) {
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index 0a9d14d6603a..ef3a18ba7df2 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -2787,6 +2787,11 @@ private:
   void handleTypesMatchConstraint(
       StringMap<TypeResolutionInstance> &variableTyResolver, const Record &def);
 
+  /// Check for inferable type resolution based on
+  /// `ShapedTypeMatchesElementCountAndTypes` constraint.
+  void handleShapedTypeMatchesElementCountAndTypesConstraint(
+      StringMap<TypeResolutionInstance> &variableTyResolver, const Record &def);
+
   /// Returns an argument or attribute with the given name that has been seen
   /// within the format.
   ConstArgument findSeenArg(StringRef name);
@@ -2850,6 +2855,9 @@ LogicalResult OpFormatParser::verify(SMLoc loc,
       handleSameTypesConstraint(variableTyResolver, /*includeResults=*/true);
     } else if (def.isSubClassOf("TypesMatchWith")) {
       handleTypesMatchConstraint(variableTyResolver, def);
+    } else if (def.isSubClassOf("ShapedTypeMatchesElementCountAndTypes")) {
+      handleShapedTypeMatchesElementCountAndTypesConstraint(variableTyResolver,
+                                                            def);
     } else if (!op.allResultTypesKnown()) {
       // This doesn't check the name directly to handle
       //    DeclareOpInterfaceMethods<InferTypeOpInterface>
@@ -3289,6 +3297,24 @@ void OpFormatParser::handleTypesMatchConstraint(
     variableTyResolver[rhsName] = {arg, transformer};
 }
 
+void OpFormatParser::handleShapedTypeMatchesElementCountAndTypesConstraint(
+    StringMap<TypeResolutionInstance> &variableTyResolver, const Record &def) {
+  StringRef shapedArg = def.getValueAsString("shaped");
+  StringRef elementsArg = def.getValueAsString("elements");
+
+  // Check if the 'shaped' argument is seen, then we can infer the 'elements'
+  // types.
+  if (ConstArgument arg = findSeenArg(shapedArg)) {
+    variableTyResolver[elementsArg] = {
+        arg, "::llvm::SmallVector<::mlir::Type>(::llvm::cast<::mlir::"
+             "ShapedType>($_self).getNumElements(), "
+             "::llvm::cast<::mlir::ShapedType>($_self).getElementType())"};
+  }
+
+  // Type inference in the opposite direction is not possible as the actual
+  // shaped type can't be inferred from the variadic elements.
+}
+
 ConstArgument OpFormatParser::findSeenArg(StringRef name) {
   if (const NamedTypeConstraint *arg = findArg(op.getOperands(), name))
     return seenOperandTypes.test(arg - op.operand_begin()) ? arg : nullptr;
-- 
cgit v1.2.3


From 86d1d6b2c0c1f03e82cb8e360f2672c6f0ea39d5 Mon Sep 17 00:00:00 2001
From: Nick Sarnie <nick.sarnie@intel.com>
Date: Wed, 18 Jun 2025 16:50:12 -0400
Subject: [clang] Use TargetInfo to determine device kernel calling convention
 (#144728)

We should abstract this logic away to `TargetInfo`. See
https://github.com/llvm/llvm-project/pull/137882 for more information.

---------

Signed-off-by: Sarnie, Nick <nick.sarnie@intel.com>
---
 clang/lib/CodeGen/CGCall.cpp         | 13 ++-----------
 clang/lib/CodeGen/TargetInfo.cpp     | 27 +++++++++++++++------------
 clang/lib/CodeGen/TargetInfo.h       |  4 ++--
 clang/lib/CodeGen/Targets/AMDGPU.cpp |  4 ++--
 clang/lib/CodeGen/Targets/NVPTX.cpp  |  2 +-
 clang/lib/CodeGen/Targets/SPIR.cpp   |  4 ++--
 6 files changed, 24 insertions(+), 30 deletions(-)

diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index a06455d25b1e..fd75de42515d 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -83,17 +83,8 @@ unsigned CodeGenTypes::ClangCallConvToLLVMCallConv(CallingConv CC) {
     return llvm::CallingConv::AArch64_SVE_VectorCall;
   case CC_SpirFunction:
     return llvm::CallingConv::SPIR_FUNC;
-  case CC_DeviceKernel: {
-    if (CGM.getLangOpts().OpenCL)
-      return CGM.getTargetCodeGenInfo().getOpenCLKernelCallingConv();
-    if (CGM.getTriple().isSPIROrSPIRV())
-      return llvm::CallingConv::SPIR_KERNEL;
-    if (CGM.getTriple().isAMDGPU())
-      return llvm::CallingConv::AMDGPU_KERNEL;
-    if (CGM.getTriple().isNVPTX())
-      return llvm::CallingConv::PTX_Kernel;
-    llvm_unreachable("Unknown kernel calling convention");
-  }
+  case CC_DeviceKernel:
+    return CGM.getTargetCodeGenInfo().getDeviceKernelCallingConv();
   case CC_PreserveMost:
     return llvm::CallingConv::PreserveMost;
   case CC_PreserveAll:
diff --git a/clang/lib/CodeGen/TargetInfo.cpp b/clang/lib/CodeGen/TargetInfo.cpp
index f3df92c44bb6..277d69daf493 100644
--- a/clang/lib/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CodeGen/TargetInfo.cpp
@@ -103,18 +103,21 @@ TargetCodeGenInfo::getDependentLibraryOption(llvm::StringRef Lib,
   Opt += Lib;
 }
 
-unsigned TargetCodeGenInfo::getOpenCLKernelCallingConv() const {
-  // OpenCL kernels are called via an explicit runtime API with arguments
-  // set with clSetKernelArg(), not as normal sub-functions.
-  // Return SPIR_KERNEL by default as the kernel calling convention to
-  // ensure the fingerprint is fixed such way that each OpenCL argument
-  // gets one matching argument in the produced kernel function argument
-  // list to enable feasible implementation of clSetKernelArg() with
-  // aggregates etc. In case we would use the default C calling conv here,
-  // clSetKernelArg() might break depending on the target-specific
-  // conventions; different targets might split structs passed as values
-  // to multiple function arguments etc.
-  return llvm::CallingConv::SPIR_KERNEL;
+unsigned TargetCodeGenInfo::getDeviceKernelCallingConv() const {
+  if (getABIInfo().getContext().getLangOpts().OpenCL) {
+    // Device kernels are called via an explicit runtime API with arguments,
+    // such as set with clSetKernelArg() for OpenCL, not as normal
+    // sub-functions. Return SPIR_KERNEL by default as the kernel calling
+    // convention to ensure the fingerprint is fixed such way that each kernel
+    // argument gets one matching argument in the produced kernel function
+    // argument list to enable feasible implementation of clSetKernelArg() with
+    // aggregates etc. In case we would use the default C calling conv here,
+    // clSetKernelArg() might break depending on the target-specific
+    // conventions; different targets might split structs passed as values
+    // to multiple function arguments etc.
+    return llvm::CallingConv::SPIR_KERNEL;
+  }
+  llvm_unreachable("Unknown kernel calling convention");
 }
 
 void TargetCodeGenInfo::setOCLKernelStubCallingConvention(
diff --git a/clang/lib/CodeGen/TargetInfo.h b/clang/lib/CodeGen/TargetInfo.h
index 2783e222eb80..b4057d369f98 100644
--- a/clang/lib/CodeGen/TargetInfo.h
+++ b/clang/lib/CodeGen/TargetInfo.h
@@ -298,8 +298,8 @@ public:
                                        llvm::StringRef Value,
                                        llvm::SmallString<32> &Opt) const {}
 
-  /// Get LLVM calling convention for OpenCL kernel.
-  virtual unsigned getOpenCLKernelCallingConv() const;
+  /// Get LLVM calling convention for device kernels.
+  virtual unsigned getDeviceKernelCallingConv() const;
 
   /// Get target specific null pointer.
   /// \param T is the LLVM type of the null pointer.
diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp
index 8660373c3927..47a552a7bf49 100644
--- a/clang/lib/CodeGen/Targets/AMDGPU.cpp
+++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp
@@ -304,7 +304,7 @@ public:
 
   void setTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
                            CodeGen::CodeGenModule &M) const override;
-  unsigned getOpenCLKernelCallingConv() const override;
+  unsigned getDeviceKernelCallingConv() const override;
 
   llvm::Constant *getNullPointer(const CodeGen::CodeGenModule &CGM,
       llvm::PointerType *T, QualType QT) const override;
@@ -431,7 +431,7 @@ void AMDGPUTargetCodeGenInfo::setTargetAttributes(
     F->addFnAttr("amdgpu-ieee", "false");
 }
 
-unsigned AMDGPUTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
+unsigned AMDGPUTargetCodeGenInfo::getDeviceKernelCallingConv() const {
   return llvm::CallingConv::AMDGPU_KERNEL;
 }
 
diff --git a/clang/lib/CodeGen/Targets/NVPTX.cpp b/clang/lib/CodeGen/Targets/NVPTX.cpp
index ad802c9131de..82bdfe2666b5 100644
--- a/clang/lib/CodeGen/Targets/NVPTX.cpp
+++ b/clang/lib/CodeGen/Targets/NVPTX.cpp
@@ -78,7 +78,7 @@ public:
     return true;
   }
 
-  unsigned getOpenCLKernelCallingConv() const override {
+  unsigned getDeviceKernelCallingConv() const override {
     return llvm::CallingConv::PTX_Kernel;
   }
 
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index 2f1e43cdc8cc..afa23bffcd07 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -51,7 +51,7 @@ public:
         getABIInfo().getDataLayout().getAllocaAddrSpace());
   }
 
-  unsigned getOpenCLKernelCallingConv() const override;
+  unsigned getDeviceKernelCallingConv() const override;
   llvm::Type *getOpenCLType(CodeGenModule &CGM, const Type *T) const override;
   llvm::Type *
   getHLSLType(CodeGenModule &CGM, const Type *Ty,
@@ -219,7 +219,7 @@ void computeSPIRKernelABIInfo(CodeGenModule &CGM, CGFunctionInfo &FI) {
 }
 }
 
-unsigned CommonSPIRTargetCodeGenInfo::getOpenCLKernelCallingConv() const {
+unsigned CommonSPIRTargetCodeGenInfo::getDeviceKernelCallingConv() const {
   return llvm::CallingConv::SPIR_KERNEL;
 }
 
-- 
cgit v1.2.3


From 03bdc0a1f68adcddef80a4e7931dbfae914e5652 Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Wed, 18 Jun 2025 13:51:59 -0700
Subject: [lldb][target] Add progress report for wait-attaching to process
 (#144768)

This commit adds a progress report when wait-attaching to a process as
well as a test for this.
---
 lldb/source/Target/Target.cpp                      |  1 +
 .../progress_reporting/TestProgressReporting.py    | 31 ++++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 45a9e1196a04..8f8d2ef21cc5 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -3546,6 +3546,7 @@ llvm::Expected<TraceSP> Target::GetTraceOrCreate() {
 }
 
 Status Target::Attach(ProcessAttachInfo &attach_info, Stream *stream) {
+  Progress attach_progress("Waiting to attach to process");
   m_stats.SetLaunchOrAttachTime();
   auto state = eStateInvalid;
   auto process_sp = GetProcessSP();
diff --git a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
index 9af53845ca1b..8198c50a5ff0 100644
--- a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
+++ b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
@@ -2,6 +2,7 @@
 Test that we are able to broadcast and receive progress events from lldb
 """
 import lldb
+import threading
 
 import lldbsuite.test.lldbutil as lldbutil
 
@@ -16,6 +17,36 @@ class TestProgressReporting(TestBase):
             self.broadcaster, lldb.SBDebugger.eBroadcastBitProgress
         )
 
+    def test_wait_attach_progress_reporting(self):
+        """Test that progress reports for wait attaching work as intended."""
+        self.build()
+        target = self.dbg.CreateTarget(None)
+
+        # Wait attach to a process, then check to see that a progress report was created
+        # and that its message is correct for waiting to attach to a process.
+        class AttachThread(threading.Thread):
+            def __init__(self, target):
+                threading.Thread.__init__(self)
+                self.target = target
+
+            def run(self):
+                self.target.AttachToProcessWithName(
+                    lldb.SBListener(), "a.out", True, lldb.SBError()
+                )
+
+        thread = AttachThread(target)
+        thread.start()
+
+        event = lldbutil.fetch_next_event(self, self.listener, self.broadcaster)
+        progress_data = lldb.SBDebugger.GetProgressDataFromEvent(event)
+        message = progress_data.GetValueForKey("message").GetStringValue(100)
+        self.assertEqual(message, "Waiting to attach to process")
+
+        # Interrupt the process attach to keep the test from stalling.
+        target.process.SendAsyncInterrupt()
+
+        thread.join()
+
     def test_dwarf_symbol_loading_progress_report(self):
         """Test that we are able to fetch dwarf symbol loading progress events"""
         self.build()
-- 
cgit v1.2.3


From 4dca4459a328b8d589d81cd1f203b798c36ebf35 Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Wed, 18 Jun 2025 23:09:48 +0200
Subject: [CIR] Upstream ComplexType builtin_complex (#144225)

This change adds support for builtin_complex

https://github.com/llvm/llvm-project/issues/141365
---
 clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp     | 11 +++++++++-
 clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp | 33 ++++++++++++++++++++++++++++-
 clang/lib/CIR/CodeGen/CIRGenValue.h         |  1 -
 clang/test/CIR/CodeGen/complex.cpp          | 25 ++++++++++++++++++++++
 4 files changed, 67 insertions(+), 3 deletions(-)

diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
index 83825f0835a1..cff139a7802d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp
@@ -78,6 +78,8 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
   assert(!cir::MissingFeatures::builtinCallMathErrno());
   assert(!cir::MissingFeatures::builtinCall());
 
+  mlir::Location loc = getLoc(e->getExprLoc());
+
   switch (builtinIDIfNoAsmLabel) {
   default:
     break;
@@ -88,9 +90,16 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl &gd, unsigned builtinID,
       return RValue::get(nullptr);
 
     mlir::Value argValue = emitCheckedArgForAssume(e->getArg(0));
-    builder.create<cir::AssumeOp>(getLoc(e->getExprLoc()), argValue);
+    builder.create<cir::AssumeOp>(loc, argValue);
     return RValue::get(nullptr);
   }
+
+  case Builtin::BI__builtin_complex: {
+    mlir::Value real = emitScalarExpr(e->getArg(0));
+    mlir::Value imag = emitScalarExpr(e->getArg(1));
+    mlir::Value complex = builder.createComplexCreate(loc, real, imag);
+    return RValue::get(complex);
+  }
   }
 
   cgm.errorNYI(e->getSourceRange(), "unimplemented builtin call");
diff --git a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
index 26070a6ca307..12e8e27948cf 100644
--- a/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenExprComplex.cpp
@@ -15,11 +15,25 @@ public:
   explicit ComplexExprEmitter(CIRGenFunction &cgf)
       : cgf(cgf), builder(cgf.getBuilder()) {}
 
+  //===--------------------------------------------------------------------===//
+  //                               Utilities
+  //===--------------------------------------------------------------------===//
+
+  /// Given an expression with complex type that represents a value l-value,
+  /// this method emits the address of the l-value, then loads and returns the
+  /// result.
+  mlir::Value emitLoadOfLValue(const Expr *e) {
+    return emitLoadOfLValue(cgf.emitLValue(e), e->getExprLoc());
+  }
+
+  mlir::Value emitLoadOfLValue(LValue lv, SourceLocation loc);
+
   /// Store the specified real/imag parts into the
   /// specified value pointer.
   void emitStoreOfComplex(mlir::Location loc, mlir::Value val, LValue lv,
                           bool isInit);
 
+  mlir::Value VisitCallExpr(const CallExpr *e);
   mlir::Value VisitInitListExpr(InitListExpr *e);
 
   mlir::Value VisitImaginaryLiteral(const ImaginaryLiteral *il);
@@ -34,11 +48,21 @@ static const ComplexType *getComplexType(QualType type) {
   return cast<ComplexType>(cast<AtomicType>(type)->getValueType());
 }
 
+mlir::Value ComplexExprEmitter::emitLoadOfLValue(LValue lv,
+                                                 SourceLocation loc) {
+  assert(lv.isSimple() && "non-simple complex l-value?");
+  if (lv.getType()->isAtomicType())
+    cgf.cgm.errorNYI(loc, "emitLoadOfLValue with Atomic LV");
+
+  const Address srcAddr = lv.getAddress();
+  return builder.createLoad(cgf.getLoc(loc), srcAddr);
+}
+
 void ComplexExprEmitter::emitStoreOfComplex(mlir::Location loc, mlir::Value val,
                                             LValue lv, bool isInit) {
   if (lv.getType()->isAtomicType() ||
       (!isInit && cgf.isLValueSuitableForInlineAtomic(lv))) {
-    cgf.cgm.errorNYI("StoreOfComplex with Atomic LV");
+    cgf.cgm.errorNYI(loc, "StoreOfComplex with Atomic LV");
     return;
   }
 
@@ -46,6 +70,13 @@ void ComplexExprEmitter::emitStoreOfComplex(mlir::Location loc, mlir::Value val,
   builder.createStore(loc, val, destAddr);
 }
 
+mlir::Value ComplexExprEmitter::VisitCallExpr(const CallExpr *e) {
+  if (e->getCallReturnType(cgf.getContext())->isReferenceType())
+    return emitLoadOfLValue(e);
+
+  return cgf.emitCallExpr(e).getValue();
+}
+
 mlir::Value ComplexExprEmitter::VisitInitListExpr(InitListExpr *e) {
   mlir::Location loc = cgf.getLoc(e->getExprLoc());
   if (e->getNumInits() == 2) {
diff --git a/clang/lib/CIR/CodeGen/CIRGenValue.h b/clang/lib/CIR/CodeGen/CIRGenValue.h
index 84972fc7f911..7180d92f8c31 100644
--- a/clang/lib/CIR/CodeGen/CIRGenValue.h
+++ b/clang/lib/CIR/CodeGen/CIRGenValue.h
@@ -88,7 +88,6 @@ public:
     return er;
   }
 
-  // FIXME: Aggregate rvalues need to retain information about whether they are
   // volatile or not.  Remove default to find all places that probably get this
   // wrong.
 
diff --git a/clang/test/CIR/CodeGen/complex.cpp b/clang/test/CIR/CodeGen/complex.cpp
index db0b9111ab4f..721db235b37d 100644
--- a/clang/test/CIR/CodeGen/complex.cpp
+++ b/clang/test/CIR/CodeGen/complex.cpp
@@ -191,6 +191,31 @@ void foo8() {
 // OGCG: store double 0.000000e+00, ptr %[[C_REAL_PTR]], align 8
 // OGCG: store double 2.000000e+00, ptr %[[C_IMAG_PTR]], align 8
 
+void foo9(double a, double b) {
+  double _Complex c = __builtin_complex(a, b);
+}
+
+// CIR: %[[INIT:.*]] = cir.alloca !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>, ["c", init]
+// CIR: %[[TMP_A:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.double>, !cir.double
+// CIR: %[[TMP_B:.*]] = cir.load{{.*}} {{.*}} : !cir.ptr<!cir.double>, !cir.double
+// CIR: %[[COMPLEX:.*]] = cir.complex.create %[[TMP_A]], %[[TMP_B]] : !cir.double -> !cir.complex<!cir.double>
+// CIR: cir.store{{.*}} %[[COMPLEX]], %[[INIT]] : !cir.complex<!cir.double>, !cir.ptr<!cir.complex<!cir.double>>
+
+// LLVM: %[[COMPLEX:.*]] = alloca { double, double }, i64 1, align 8
+// LLVM: %[[TMP_A:.*]] = load double, ptr {{.*}}, align 8
+// LLVM: %[[TMP_B:.*]] = load double, ptr {{.*}}, align 8
+// LLVM: %[[TMP:.*]] = insertvalue { double, double } undef, double %[[TMP_A]], 0
+// LLVM: %[[TMP_2:.*]] = insertvalue { double, double } %[[TMP]], double %[[TMP_B]], 1
+// LLVM: store { double, double } %[[TMP_2]], ptr %[[COMPLEX]], align 8
+
+// OGCG: %[[COMPLEX]] = alloca { double, double }, align 8
+// OGCG: %[[TMP_A:.*]] = load double, ptr {{.*}}, align 8
+// OGCG: %[[TMP_B:.*]] = load double, ptr {{.*}}, align 8
+// OGCG: %[[C_REAL_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX]], i32 0, i32 0
+// OGCG: %[[C_IMAG_PTR:.*]] = getelementptr inbounds nuw { double, double }, ptr %[[COMPLEX]], i32 0, i32 1
+// OGCG: store double %[[TMP_A]], ptr %[[C_REAL_PTR]], align 8
+// OGCG: store double %[[TMP_B]], ptr %[[C_IMAG_PTR]], align 8
+
 void foo14() {
   int _Complex c = 2i;
 }
-- 
cgit v1.2.3


From ac37a0df949afc31d12de75f85306db32dd50713 Mon Sep 17 00:00:00 2001
From: Diego Caballero <dieg0ca6aller0@gmail.com>
Date: Wed, 18 Jun 2025 14:11:21 -0700
Subject: [mlir] Fix integer comparison warning (#144794)

Introduced by https://github.com/llvm/llvm-project/pull/141457
---
 mlir/include/mlir/IR/OpBase.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/IR/OpBase.td b/mlir/include/mlir/IR/OpBase.td
index b3fabe409806..43ef28624fb1 100644
--- a/mlir/include/mlir/IR/OpBase.td
+++ b/mlir/include/mlir/IR/OpBase.td
@@ -566,7 +566,7 @@ class ShapedTypeMatchesElementCountAndTypes<string shapedArg,
     PredOpTrait<"shaped type '" # shapedArg # "' matches '" # elementsArg # "' "
                 "element count and types",
         And<[CPred<ElementCount<shapedArg>.result # " == "
-                     "$" # elementsArg # ".getTypes().size()">,
+                     "static_cast<int64_t>($" # elementsArg # ".getTypes().size())">,
              CPred<"::llvm::all_of($" # elementsArg # ".getTypes(), "
                      "[&](::mlir::Type t) { return t == "
                        # ElementType<shapedArg>.result # "; })">]>> {
-- 
cgit v1.2.3


From d10079e305acae58b44dc773cb94f7127de197ef Mon Sep 17 00:00:00 2001
From: Min-Yih Hsu <min.hsu@sifive.com>
Date: Wed, 18 Jun 2025 14:23:51 -0700
Subject: [RISCV] Reduce the VL of both operands in VMERGE_VVM (#144759)

The `tryToReduceVL` function in RISCVVectorPeephole currently only
reduces the VL of the instruction that defines the true operand in
VMERGE_VVM. We should be able to reduce VL of both operands. This patch
generalizes this function to support multiple operands from a single
instruction.
---
 llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp  | 74 ++++++++++++++------------
 llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll |  3 +-
 llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll   |  4 +-
 3 files changed, 43 insertions(+), 38 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
index c9c2413d009b..f7acd676461f 100644
--- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
+++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp
@@ -112,7 +112,7 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
   //
   // TODO: We can handle a bunch more instructions here, and probably
   // recurse backwards through operands too.
-  unsigned SrcIdx = 0;
+  SmallVector<unsigned, 2> SrcIndices = {0};
   switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {
   default:
     return false;
@@ -122,10 +122,10 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
   case RISCV::VSE64_V:
     break;
   case RISCV::VMV_V_V:
-    SrcIdx = 2;
+    SrcIndices[0] = 2;
     break;
   case RISCV::VMERGE_VVM:
-    SrcIdx = 3; // TODO: We can also handle the false operand.
+    SrcIndices.assign({2, 3});
     break;
   case RISCV::VREDSUM_VS:
   case RISCV::VREDMAXU_VS:
@@ -143,7 +143,7 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
   case RISCV::VFREDMIN_VS:
   case RISCV::VFWREDUSUM_VS:
   case RISCV::VFWREDOSUM_VS:
-    SrcIdx = 2;
+    SrcIndices[0] = 2;
     break;
   }
 
@@ -151,42 +151,48 @@ bool RISCVVectorPeephole::tryToReduceVL(MachineInstr &MI) const {
   if (VL.isImm() && VL.getImm() == RISCV::VLMaxSentinel)
     return false;
 
-  Register SrcReg = MI.getOperand(SrcIdx).getReg();
-  // Note: one *use*, not one *user*.
-  if (!MRI->hasOneUse(SrcReg))
-    return false;
-
-  MachineInstr *Src = MRI->getVRegDef(SrcReg);
-  if (!Src || Src->hasUnmodeledSideEffects() ||
-      Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||
-      !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||
-      !RISCVII::hasSEWOp(Src->getDesc().TSFlags))
-    return false;
-
-  // Src's dest needs to have the same EEW as MI's input.
-  if (!hasSameEEW(MI, *Src))
-    return false;
-
-  bool ElementsDependOnVL = RISCVII::elementsDependOnVL(
-      TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);
-  if (ElementsDependOnVL || Src->mayRaiseFPException())
-    return false;
+  bool Changed = false;
+  for (unsigned SrcIdx : SrcIndices) {
+    Register SrcReg = MI.getOperand(SrcIdx).getReg();
+    // Note: one *use*, not one *user*.
+    if (!MRI->hasOneUse(SrcReg))
+      continue;
+
+    MachineInstr *Src = MRI->getVRegDef(SrcReg);
+    if (!Src || Src->hasUnmodeledSideEffects() ||
+        Src->getParent() != MI.getParent() || Src->getNumDefs() != 1 ||
+        !RISCVII::hasVLOp(Src->getDesc().TSFlags) ||
+        !RISCVII::hasSEWOp(Src->getDesc().TSFlags))
+      continue;
+
+    // Src's dest needs to have the same EEW as MI's input.
+    if (!hasSameEEW(MI, *Src))
+      continue;
+
+    bool ElementsDependOnVL = RISCVII::elementsDependOnVL(
+        TII->get(RISCV::getRVVMCOpcode(Src->getOpcode())).TSFlags);
+    if (ElementsDependOnVL || Src->mayRaiseFPException())
+      continue;
+
+    MachineOperand &SrcVL =
+        Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
+    if (VL.isIdenticalTo(SrcVL) || !RISCV::isVLKnownLE(VL, SrcVL))
+      continue;
 
-  MachineOperand &SrcVL = Src->getOperand(RISCVII::getVLOpNum(Src->getDesc()));
-  if (VL.isIdenticalTo(SrcVL) || !RISCV::isVLKnownLE(VL, SrcVL))
-    return false;
+    if (!ensureDominates(VL, *Src))
+      continue;
 
-  if (!ensureDominates(VL, *Src))
-    return false;
+    if (VL.isImm())
+      SrcVL.ChangeToImmediate(VL.getImm());
+    else if (VL.isReg())
+      SrcVL.ChangeToRegister(VL.getReg(), false);
 
-  if (VL.isImm())
-    SrcVL.ChangeToImmediate(VL.getImm());
-  else if (VL.isReg())
-    SrcVL.ChangeToRegister(VL.getReg(), false);
+    Changed = true;
+  }
 
   // TODO: For instructions with a passthru, we could clear the passthru
   // and tail policy since we've just proven the tail is not demanded.
-  return true;
+  return Changed;
 }
 
 /// Check if an operand is an immediate or a materialized ADDI $x0, imm.
diff --git a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
index 75537406f351..372b07e0137b 100644
--- a/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/masked-load-int.ll
@@ -34,9 +34,8 @@ define <vscale x 1 x i8> @masked_load_passthru_nxv1i8(ptr %a, <vscale x 1 x i1>
 ; ZVE32:       # %bb.0:
 ; ZVE32-NEXT:    csrr a1, vlenb
 ; ZVE32-NEXT:    srli a1, a1, 3
-; ZVE32-NEXT:    vsetvli a2, zero, e8, mf4, ta, ma
-; ZVE32-NEXT:    vmv.v.i v8, 0
 ; ZVE32-NEXT:    vsetvli zero, a1, e8, mf4, ta, mu
+; ZVE32-NEXT:    vmv.v.i v8, 0
 ; ZVE32-NEXT:    vle8.v v8, (a0), v0.t
 ; ZVE32-NEXT:    ret
   %load = call <vscale x 1 x i8> @llvm.masked.load.nxv1i8(ptr %a, i32 1, <vscale x 1 x i1> %mask, <vscale x 1 x i8> zeroinitializer)
diff --git a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
index 10a92f0188a9..1cbb980aebff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vl-opt-instrs.ll
@@ -3063,9 +3063,9 @@ define <vscale x 4 x i32> @vmv_v_x(<vscale x 4 x i32> %a, i32 %x, iXLen %vl) {
 define <vscale x 1 x i8> @vmv_v_v(<vscale x 1 x i8> %a, <vscale x 1 x i8> %b, <vscale x 1 x i8> %c, <vscale x 1 x i1> %m, iXLen %vl) {
 ; NOVLOPT-LABEL: vmv_v_v:
 ; NOVLOPT:       # %bb.0:
-; NOVLOPT-NEXT:    vsetvli a1, zero, e8, mf8, tu, ma
+; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, tu, ma
 ; NOVLOPT-NEXT:    vmv.v.v v8, v9
-; NOVLOPT-NEXT:    vsetvli zero, a0, e8, mf8, ta, ma
+; NOVLOPT-NEXT:    vsetvli zero, zero, e8, mf8, ta, ma
 ; NOVLOPT-NEXT:    vmerge.vvm v8, v8, v10, v0
 ; NOVLOPT-NEXT:    ret
 ;
-- 
cgit v1.2.3


From c4d7ea8049688a1d6d6d93129893fd1700a9f7e5 Mon Sep 17 00:00:00 2001
From: Javier Lopez-Gomez <javier.lopez.gomez@proton.me>
Date: Wed, 18 Jun 2025 23:47:30 +0200
Subject: [llvm-debuginfo-analyzer] Apply various memory savings in Core/LVxxx
 base classes (#144399)

This small changelist reduces memory footprint of instances of the Core
classes. Specifically,

- For `LVProperties`, use underlying type of `uint32_t` if there are at
most 32 properties to keep track of. Otherwise, fallback to the generic
`std::bitset<N>`.
The use of `llvm::SmallBitVector` is disregarded in this case, as the
upper bound on the size of the bitset can be determined statically (no
heap alloc ever needed).
- Reorder members in `LVElement` s.t. padding between members is
reduced.
- `LVScopeCompileUnit`: fix a couple of members which should be `static
constexpr` instead.
---
 .../llvm/DebugInfo/LogicalView/Core/LVElement.h    | 10 ++++----
 .../llvm/DebugInfo/LogicalView/Core/LVScope.h      |  4 +--
 .../llvm/DebugInfo/LogicalView/Core/LVSupport.h    | 29 ++++++++++++++++++----
 3 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
index b4501db190fe..0e7be45abfef 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVElement.h
@@ -107,18 +107,18 @@ class LLVM_ABI LVElement : public LVObject {
     IsAnonymous,
     LastEntry
   };
-  // Typed bitvector with properties for this element.
-  LVProperties<Property> Properties;
   static LVElementDispatch Dispatch;
 
-  /// RTTI.
-  const LVSubclassID SubclassID;
-
   // Indexes in the String Pool.
   size_t NameIndex = 0;
   size_t QualifiedNameIndex = 0;
   size_t FilenameIndex = 0;
 
+  // Typed bitvector with properties for this element.
+  LVProperties<Property> Properties;
+  /// RTTI.
+  const LVSubclassID SubclassID;
+
   uint16_t AccessibilityCode : 2; // DW_AT_accessibility.
   uint16_t InlineCode : 2;        // DW_AT_inline.
   uint16_t VirtualityCode : 2;    // DW_AT_virtuality.
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
index 5715a37185b2..a453923d032e 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVScope.h
@@ -473,7 +473,7 @@ class LLVM_ABI LVScopeCompileUnit final : public LVScope {
 
   // Record scope sizes indexed by lexical level.
   // Setting an initial size that will cover a very deep nested scopes.
-  const size_t TotalInitialSize = 8;
+  static constexpr size_t TotalInitialSize = 8;
   using LVTotalsEntry = std::pair<unsigned, float>;
   SmallVector<LVTotalsEntry> Totals;
   // Maximum seen lexical level. It is used to control how many entries
@@ -510,7 +510,7 @@ public:
   void addMapping(LVLine *Line, LVSectionIndex SectionIndex);
   LVLineRange lineRange(LVLocation *Location) const;
 
-  LVNameInfo NameNone = {UINT64_MAX, 0};
+  static constexpr LVNameInfo NameNone = {UINT64_MAX, 0};
   void addPublicName(LVScope *Scope, LVAddress LowPC, LVAddress HighPC) {
     PublicNames.emplace(std::piecewise_construct, std::forward_as_tuple(Scope),
                         std::forward_as_tuple(LowPC, HighPC - LowPC));
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
index 01137f80c0f8..058ca2da9a96 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSupport.h
@@ -13,7 +13,6 @@
 #ifndef LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSUPPORT_H
 #define LLVM_DEBUGINFO_LOGICALVIEW_CORE_LVSUPPORT_H
 
-#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/DebugInfo/LogicalView/Core/LVStringPool.h"
 #include "llvm/Support/Compiler.h"
@@ -21,9 +20,11 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
+#include <bitset>
 #include <cctype>
 #include <map>
 #include <sstream>
+#include <type_traits>
 
 namespace llvm {
 namespace logicalview {
@@ -38,14 +39,32 @@ using LVLexicalIndex =
 
 // Used to record specific characteristics about the objects.
 template <typename T> class LVProperties {
-  SmallBitVector Bits = SmallBitVector(static_cast<unsigned>(T::LastEntry) + 1);
+  static constexpr unsigned N_PROPS = static_cast<unsigned>(T::LastEntry);
+  // Use uint32_t as the underlying type if the `T` enum has at most 32
+  // enumerators; otherwise, fallback to the generic `std::bitset` case.
+  std::conditional_t<(N_PROPS > 32), std::bitset<N_PROPS>, uint32_t> Bits{};
 
 public:
   LVProperties() = default;
 
-  void set(T Idx) { Bits[static_cast<unsigned>(Idx)] = 1; }
-  void reset(T Idx) { Bits[static_cast<unsigned>(Idx)] = 0; }
-  bool get(T Idx) const { return Bits[static_cast<unsigned>(Idx)]; }
+  void set(T Idx) {
+    if constexpr (std::is_same_v<decltype(Bits), uint32_t>)
+      Bits |= 1 << static_cast<unsigned>(Idx);
+    else
+      Bits.set(static_cast<unsigned>(Idx));
+  }
+  void reset(T Idx) {
+    if constexpr (std::is_same_v<decltype(Bits), uint32_t>)
+      Bits &= ~(1 << static_cast<unsigned>(Idx));
+    else
+      Bits.reset(static_cast<unsigned>(Idx));
+  }
+  bool get(T Idx) const {
+    if constexpr (std::is_same_v<decltype(Bits), uint32_t>)
+      return Bits & (1 << static_cast<unsigned>(Idx));
+    else
+      return Bits[static_cast<unsigned>(Idx)];
+  }
 };
 
 // Generate get, set and reset 'bool' functions for LVProperties instances.
-- 
cgit v1.2.3


From 51aa6a4993ea18c968a087352d1cf569c840c41f Mon Sep 17 00:00:00 2001
From: Ebuka Ezike <yerimyah1@gmail.com>
Date: Wed, 18 Jun 2025 22:48:24 +0100
Subject: [lldb-dap] Use protocol types for ReadMemory request (#144552)

Read memory from process instead of target.
---
 .../API/tools/lldb-dap/memory/TestDAP_memory.py    |  17 ++-
 .../lldb-dap/Handler/ReadMemoryRequestHandler.cpp  | 143 ++++-----------------
 lldb/tools/lldb-dap/Handler/RequestHandler.h       |   9 +-
 lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp  |  37 ++++++
 lldb/tools/lldb-dap/Protocol/ProtocolRequests.h    |  36 ++++++
 lldb/unittests/DAP/ProtocolTypesTest.cpp           |  33 +++++
 6 files changed, 152 insertions(+), 123 deletions(-)

diff --git a/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
index 74062f3ab216..55fb4a961e78 100644
--- a/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
+++ b/lldb/test/API/tools/lldb-dap/memory/TestDAP_memory.py
@@ -111,8 +111,17 @@ class TestDAP_memory(lldbdap_testcase.DAPTestCaseBase):
         # VS Code sends those in order to check if a `memoryReference` can actually be dereferenced.
         mem = self.dap_server.request_readMemory(memref, 0, 0)
         self.assertEqual(mem["success"], True)
-        self.assertEqual(mem["body"]["data"], "")
+        self.assertNotIn(
+            "data", mem["body"], f"expects no data key in response: {mem!r}"
+        )
+
+        # Reads at offset 0x0 return unreadable bytes
+        bytes_to_read = 6
+        mem = self.dap_server.request_readMemory("0x0", 0, bytes_to_read)
+        self.assertEqual(mem["body"]["unreadableBytes"], bytes_to_read)
+
+        # Reads with invalid address fails.
+        mem = self.dap_server.request_readMemory("-3204", 0, 10)
+        self.assertFalse(mem["success"], "expect fail on reading memory.")
 
-        # Reads at offset 0x0 fail
-        mem = self.dap_server.request_readMemory("0x0", 0, 6)
-        self.assertEqual(mem["success"], False)
+        self.continue_to_exit()
diff --git a/lldb/tools/lldb-dap/Handler/ReadMemoryRequestHandler.cpp b/lldb/tools/lldb-dap/Handler/ReadMemoryRequestHandler.cpp
index 891c2af4f2f2..7065b6a24b55 100644
--- a/lldb/tools/lldb-dap/Handler/ReadMemoryRequestHandler.cpp
+++ b/lldb/tools/lldb-dap/Handler/ReadMemoryRequestHandler.cpp
@@ -7,136 +7,47 @@
 //===----------------------------------------------------------------------===//
 
 #include "DAP.h"
-#include "EventHelper.h"
 #include "JSONUtils.h"
 #include "RequestHandler.h"
 #include "llvm/ADT/StringExtras.h"
-#include "llvm/Support/Base64.h"
 
 namespace lldb_dap {
 
-// "ReadMemoryRequest": {
-//   "allOf": [ { "$ref": "#/definitions/Request" }, {
-//     "type": "object",
-//     "description": "Reads bytes from memory at the provided location. Clients
-//                     should only call this request if the corresponding
-//                     capability `supportsReadMemoryRequest` is true.",
-//     "properties": {
-//       "command": {
-//         "type": "string",
-//         "enum": [ "readMemory" ]
-//       },
-//       "arguments": {
-//         "$ref": "#/definitions/ReadMemoryArguments"
-//       }
-//     },
-//     "required": [ "command", "arguments" ]
-//   }]
-// },
-// "ReadMemoryArguments": {
-//   "type": "object",
-//   "description": "Arguments for `readMemory` request.",
-//   "properties": {
-//     "memoryReference": {
-//       "type": "string",
-//       "description": "Memory reference to the base location from which data
-//                       should be read."
-//     },
-//     "offset": {
-//       "type": "integer",
-//       "description": "Offset (in bytes) to be applied to the reference
-//                       location before reading data. Can be negative."
-//     },
-//     "count": {
-//       "type": "integer",
-//       "description": "Number of bytes to read at the specified location and
-//                       offset."
-//     }
-//   },
-//   "required": [ "memoryReference", "count" ]
-// },
-// "ReadMemoryResponse": {
-//   "allOf": [ { "$ref": "#/definitions/Response" }, {
-//     "type": "object",
-//     "description": "Response to `readMemory` request.",
-//     "properties": {
-//       "body": {
-//         "type": "object",
-//         "properties": {
-//           "address": {
-//             "type": "string",
-//             "description": "The address of the first byte of data returned.
-//                             Treated as a hex value if prefixed with `0x`, or
-//                             as a decimal value otherwise."
-//           },
-//           "unreadableBytes": {
-//             "type": "integer",
-//             "description": "The number of unreadable bytes encountered after
-//                             the last successfully read byte.\nThis can be
-//                             used to determine the number of bytes that should
-//                             be skipped before a subsequent
-//             `readMemory` request succeeds."
-//           },
-//           "data": {
-//             "type": "string",
-//             "description": "The bytes read from memory, encoded using base64.
-//                             If the decoded length of `data` is less than the
-//                             requested `count` in the original `readMemory`
-//                             request, and `unreadableBytes` is zero or
-//                             omitted, then the client should assume it's
-//                             reached the end of readable memory."
-//           }
-//         },
-//         "required": [ "address" ]
-//       }
-//     }
-//   }]
-// },
-void ReadMemoryRequestHandler::operator()(
-    const llvm::json::Object &request) const {
-  llvm::json::Object response;
-  FillResponse(request, response);
-  auto *arguments = request.getObject("arguments");
+// Reads bytes from memory at the provided location.
+//
+// Clients should only call this request if the corresponding capability
+// `supportsReadMemoryRequest` is true
+llvm::Expected<protocol::ReadMemoryResponseBody>
+ReadMemoryRequestHandler::Run(const protocol::ReadMemoryArguments &args) const {
+  const lldb::addr_t raw_address = args.memoryReference + args.offset;
 
-  llvm::StringRef memoryReference =
-      GetString(arguments, "memoryReference").value_or("");
-  auto addr_opt = DecodeMemoryReference(memoryReference);
-  if (!addr_opt.has_value()) {
-    response["success"] = false;
-    response["message"] =
-        "Malformed memory reference: " + memoryReference.str();
-    dap.SendJSON(llvm::json::Value(std::move(response)));
-    return;
-  }
-  lldb::addr_t addr_int = *addr_opt;
-  addr_int += GetInteger<uint64_t>(arguments, "offset").value_or(0);
-  const uint64_t count_requested =
-      GetInteger<uint64_t>(arguments, "count").value_or(0);
+  lldb::SBProcess process = dap.target.GetProcess();
+  if (!lldb::SBDebugger::StateIsStoppedState(process.GetState()))
+    return llvm::make_error<NotStoppedError>();
 
+  const uint64_t count_read = std::max<uint64_t>(args.count, 1);
   // We also need support reading 0 bytes
   // VS Code sends those requests to check if a `memoryReference`
   // can be dereferenced.
-  const uint64_t count_read = std::max<uint64_t>(count_requested, 1);
-  std::vector<uint8_t> buf;
-  buf.resize(count_read);
+  protocol::ReadMemoryResponseBody response;
+  std::vector<std::byte> &buffer = response.data;
+  buffer.resize(count_read);
+
   lldb::SBError error;
-  lldb::SBAddress addr{addr_int, dap.target};
-  size_t count_result =
-      dap.target.ReadMemory(addr, buf.data(), count_read, error);
-  if (count_result == 0) {
-    response["success"] = false;
-    EmplaceSafeString(response, "message", error.GetCString());
-    dap.SendJSON(llvm::json::Value(std::move(response)));
-    return;
+  const size_t memory_count = dap.target.GetProcess().ReadMemory(
+      raw_address, buffer.data(), buffer.size(), error);
+
+  response.address = "0x" + llvm::utohexstr(raw_address);
+
+  // reading memory may fail for multiple reasons. memory not readable,
+  // reading out of memory range and gaps in memory. return from
+  // the last readable byte.
+  if (error.Fail() && (memory_count < count_read)) {
+    response.unreadableBytes = count_read - memory_count;
   }
-  buf.resize(std::min<size_t>(count_result, count_requested));
 
-  llvm::json::Object body;
-  std::string formatted_addr = "0x" + llvm::utohexstr(addr_int);
-  body.try_emplace("address", formatted_addr);
-  body.try_emplace("data", llvm::encodeBase64(buf));
-  response.try_emplace("body", std::move(body));
-  dap.SendJSON(llvm::json::Value(std::move(response)));
+  buffer.resize(std::min<size_t>(memory_count, args.count));
+  return response;
 }
 
 } // namespace lldb_dap
diff --git a/lldb/tools/lldb-dap/Handler/RequestHandler.h b/lldb/tools/lldb-dap/Handler/RequestHandler.h
index 054cc7a32131..e35b9830ab60 100644
--- a/lldb/tools/lldb-dap/Handler/RequestHandler.h
+++ b/lldb/tools/lldb-dap/Handler/RequestHandler.h
@@ -564,14 +564,17 @@ public:
   Run(const protocol::DisassembleArguments &args) const override;
 };
 
-class ReadMemoryRequestHandler : public LegacyRequestHandler {
+class ReadMemoryRequestHandler final
+    : public RequestHandler<protocol::ReadMemoryArguments,
+                            llvm::Expected<protocol::ReadMemoryResponseBody>> {
 public:
-  using LegacyRequestHandler::LegacyRequestHandler;
+  using RequestHandler::RequestHandler;
   static llvm::StringLiteral GetCommand() { return "readMemory"; }
   FeatureSet GetSupportedFeatures() const override {
     return {protocol::eAdapterFeatureReadMemoryRequest};
   }
-  void operator()(const llvm::json::Object &request) const override;
+  llvm::Expected<protocol::ReadMemoryResponseBody>
+  Run(const protocol::ReadMemoryArguments &args) const override;
 };
 
 class CancelRequestHandler : public RequestHandler<protocol::CancelArguments,
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
index e6ba54ed4dcd..9bd84a6c898f 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.cpp
@@ -7,9 +7,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "Protocol/ProtocolRequests.h"
+#include "JSONUtils.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Base64.h"
 #include "llvm/Support/JSON.h"
 #include <utility>
 
@@ -480,4 +482,39 @@ json::Value toJSON(const DisassembleResponseBody &DRB) {
   return json::Object{{"instructions", DRB.instructions}};
 }
 
+bool fromJSON(const json::Value &Params, ReadMemoryArguments &RMA,
+              json::Path P) {
+  json::ObjectMapper O(Params, P);
+
+  const json::Object *rma_obj = Params.getAsObject();
+  constexpr llvm::StringRef ref_key = "memoryReference";
+  const std::optional<llvm::StringRef> memory_ref = rma_obj->getString(ref_key);
+  if (!memory_ref) {
+    P.field(ref_key).report("missing value");
+    return false;
+  }
+
+  const std::optional<lldb::addr_t> addr_opt =
+      DecodeMemoryReference(*memory_ref);
+  if (!addr_opt) {
+    P.field(ref_key).report("Malformed memory reference");
+    return false;
+  }
+
+  RMA.memoryReference = *addr_opt;
+
+  return O && O.map("count", RMA.count) && O.mapOptional("offset", RMA.offset);
+}
+
+json::Value toJSON(const ReadMemoryResponseBody &RMR) {
+  json::Object result{{"address", RMR.address}};
+
+  if (RMR.unreadableBytes != 0)
+    result.insert({"unreadableBytes", RMR.unreadableBytes});
+  if (!RMR.data.empty())
+    result.insert({"data", llvm::encodeBase64(RMR.data)});
+
+  return result;
+}
+
 } // namespace lldb_dap::protocol
diff --git a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
index 01b8f2445c9f..7d9a99fdacce 100644
--- a/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
+++ b/lldb/tools/lldb-dap/Protocol/ProtocolRequests.h
@@ -839,6 +839,42 @@ bool fromJSON(const llvm::json::Value &, DisassembleResponseBody &,
               llvm::json::Path);
 llvm::json::Value toJSON(const DisassembleResponseBody &);
 
+/// Arguments for `readMemory` request.
+struct ReadMemoryArguments {
+  /// Memory reference to the base location from which data should be read.
+  lldb::addr_t memoryReference;
+
+  /// Offset (in bytes) to be applied to the reference location before reading
+  /// data. Can be negative.
+  int64_t offset = 0;
+
+  /// Number of bytes to read at the specified location and offset.
+  uint64_t count;
+};
+bool fromJSON(const llvm::json::Value &, ReadMemoryArguments &,
+              llvm::json::Path);
+
+/// Response to `readMemory` request.
+struct ReadMemoryResponseBody {
+  /// The address of the first byte of data returned.
+  /// Treated as a hex value if prefixed with `0x`, or as a decimal value
+  /// otherwise.
+  std::string address;
+
+  /// The number of unreadable bytes encountered after the last successfully
+  /// read byte.
+  /// This can be used to determine the number of bytes that should be skipped
+  /// before a subsequent `readMemory` request succeeds.
+  uint64_t unreadableBytes = 0;
+
+  /// The bytes read from memory, encoded using base64. If the decoded length
+  /// of `data` is less than the requested `count` in the original `readMemory`
+  /// request, and `unreadableBytes` is zero or omitted, then the client should
+  /// assume it's reached the end of readable memory.
+  std::vector<std::byte> data;
+};
+llvm::json::Value toJSON(const ReadMemoryResponseBody &);
+
 } // namespace lldb_dap::protocol
 
 #endif
diff --git a/lldb/unittests/DAP/ProtocolTypesTest.cpp b/lldb/unittests/DAP/ProtocolTypesTest.cpp
index 46a09f090fea..9c93eb8c94b0 100644
--- a/lldb/unittests/DAP/ProtocolTypesTest.cpp
+++ b/lldb/unittests/DAP/ProtocolTypesTest.cpp
@@ -765,3 +765,36 @@ TEST(ProtocolTypesTest, StepInTarget) {
   EXPECT_EQ(target.endLine, deserialized_target->endLine);
   EXPECT_EQ(target.endColumn, deserialized_target->endColumn);
 }
+
+TEST(ProtocolTypesTest, ReadMemoryArguments) {
+  ReadMemoryArguments args;
+  args.count = 20;
+  args.memoryReference = 43962;
+  args.offset = 0;
+
+  llvm::Expected<ReadMemoryArguments> expected =
+      parse<ReadMemoryArguments>(R"({"memoryReference":"-4000", "count": 20})");
+  ASSERT_THAT_EXPECTED(expected, llvm::Failed());
+  expected = parse<ReadMemoryArguments>(
+      R"({"memoryReference":"0xabba", "count": 20})");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+
+  EXPECT_EQ(args.count, expected->count);
+  EXPECT_EQ(args.memoryReference, expected->memoryReference);
+  EXPECT_EQ(args.offset, expected->offset);
+}
+
+TEST(ProtocolTypesTest, ReadMemoryResponseBody) {
+  ReadMemoryResponseBody response;
+  response.address = "0xdeadbeef";
+  const std::string data_str = "hello world!";
+  std::transform(data_str.begin(), data_str.end(),
+                 std::back_inserter(response.data),
+                 [](char letter) { return std::byte(letter); });
+  response.unreadableBytes = 1;
+
+  Expected<Value> expected = json::parse(
+      R"({ "address": "0xdeadbeef", "data": "aGVsbG8gd29ybGQh", "unreadableBytes": 1})");
+  ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
+  EXPECT_EQ(pp(*expected), pp(response));
+}
\ No newline at end of file
-- 
cgit v1.2.3


From 118bfcda46c17349575217bc901e8e5942521955 Mon Sep 17 00:00:00 2001
From: Jianhui Li <jian.hui.li@intel.com>
Date: Wed, 18 Jun 2025 14:52:03 -0700
Subject: [MLIR][XEGPU] Add blocking support for scatter ops (#144766)

Add blocking support for scatter ops: Create_tdesc, update, prefetch,
load and store. It also enables the load/store with chunk size.
---
 .../lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp |  42 ++++++--
 mlir/test/Dialect/XeGPU/xegpu-blocking.mlir        | 113 +++++++++++++++++++--
 .../test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp |  14 ++-
 3 files changed, 142 insertions(+), 27 deletions(-)

diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
index a3826c56e1f6..3950e8f70d1c 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUBlocking.cpp
@@ -134,11 +134,13 @@ XeGPUBlockingPass::getTileShape(const T &operandOrResult) const {
 
 std::optional<SmallVector<int64_t>>
 XeGPUBlockingPass::getTileShape(Operation *op) const {
-  if (isa<xegpu::CreateNdDescOp, xegpu::UpdateNdOffsetOp>(op))
+  if (isa<xegpu::CreateNdDescOp, xegpu::UpdateNdOffsetOp, xegpu::CreateDescOp,
+          xegpu::UpdateOffsetOp>(op))
     return getTileShape(op->getOpResult(0));
-  if (isa<xegpu::PrefetchNdOp, xegpu::LoadNdOp>(op))
+  if (isa<xegpu::PrefetchNdOp, xegpu::LoadNdOp, xegpu::PrefetchOp,
+          xegpu::LoadGatherOp>(op))
     return getTileShape(op->getOpOperand(0));
-  if (isa<xegpu::StoreNdOp>(op))
+  if (isa<xegpu::StoreNdOp, xegpu::StoreScatterOp>(op))
     return getTileShape(op->getOpOperand(1));
 
   if (isa<xegpu::DpasOp>(op)) {
@@ -295,12 +297,36 @@ void XeGPUBlockingPass::runOnOperation() {
     Type elemTy = type.getElementType();
     Type newTy;
 
-    if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(type))
-      newTy = xegpu::TensorDescType::get(
-          ctx, tileShape, elemTy, tdescTy.getEncoding(),
-          tdescTy.getLayoutAttr().dropInstData());
-    else
+    if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(type)) {
+
+      Attribute encoding = tdescTy.getEncoding();
+      // If the encoding is a ScatterTensorDescAttr, we need to
+      // potentially adjust the chunk size based on the inst_data.
+      if (tdescTy.isScattered()) {
+        auto scatterAttr =
+            llvm::dyn_cast_if_present<xegpu::ScatterTensorDescAttr>(encoding);
+        int64_t chunkSize = scatterAttr.getChunkSize().getInt();
+
+        if (chunkSize > 1) {
+          int64_t blockedChunkSize = chunkSize;
+          auto instData = tdescTy.getLayoutAttr().getInstData();
+          if (!instData.empty())
+            blockedChunkSize = instData.asArrayRef().back();
+
+          // To create a new attribute with a different chunk_size:
+          auto newEncoding = xegpu::ScatterTensorDescAttr::get(
+              ctx, scatterAttr.getMemorySpace().getValue(), blockedChunkSize);
+
+          encoding = newEncoding;
+        }
+      }
+
+      newTy =
+          xegpu::TensorDescType::get(ctx, tileShape, elemTy, encoding,
+                                     tdescTy.getLayoutAttr().dropInstData());
+    } else {
       newTy = type.clone(tileShape, elemTy);
+    }
 
     std::optional<SmallVector<int64_t>> ratio =
         computeShapeRatio(type.getShape(), tileShape);
diff --git a/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir b/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
index 67d3bd9b393c..f977ba3c11bc 100644
--- a/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
+++ b/mlir/test/Dialect/XeGPU/xegpu-blocking.mlir
@@ -250,8 +250,7 @@ gpu.module @test_kernel {
 // -----
 #l = #xegpu.layout<inst_data = [16, 16]>
 #r = #xegpu.layout<inst_data = [16]>
-
-gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+gpu.module @test_kernel  {
   gpu.func @reduce_dim_0(%a: memref<16x512xf32>, %b: memref<512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
     %acc = arith.constant dense<0.0> : vector<64xf32>
     %c64 = arith.constant 64 : index
@@ -271,8 +270,7 @@ gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<
 // -----
 #l = #xegpu.layout<inst_data = [16, 16]>
 #r = #xegpu.layout<inst_data = [16]>
-
-gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+gpu.module @test_kernel   {
   gpu.func @reduce_dim_1(%a: memref<512x32xf32>, %b: memref<512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
     %c1 = arith.constant 1 : index
     %c32 = arith.constant 32 : index
@@ -299,8 +297,7 @@ gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<
 // -----
 #r = #xegpu.layout<inst_data = [16]>
 #l = #xegpu.layout<inst_data = [16, 16]>
-
-gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+gpu.module @test_kernel   {
   gpu.func @broadcast_dim_0(%a: memref<512xf32>, %b: memref<16x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
 
     %c64 = arith.constant 64 : index
@@ -319,8 +316,7 @@ gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<
 // -----
 #r = #xegpu.layout<inst_data = [16]>
 #l = #xegpu.layout<inst_data = [16, 16]>
-
-gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+gpu.module @test_kernel  {
   gpu.func @broadcast_dim_1(%a: memref<512xf32>, %b: memref<16x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
 
     %c32 = arith.constant 32 : index
@@ -340,8 +336,7 @@ gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<
 // -----
 #l = #xegpu.layout<inst_data = [16, 8]>
 #t = #xegpu.layout<inst_data = [8, 16]>
-
-gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Addresses, Float16Buffer, Int64, Int16, Int8, Kernel, Linkage, Vector16, GenericPointer, Groups, Float16, Float64, AtomicFloat32AddEXT, ExpectAssumeKHR, SubgroupDispatch, VectorComputeINTEL, VectorAnyINTEL], [SPV_EXT_shader_atomic_float_add, SPV_KHR_expect_assume, SPV_INTEL_vector_compute]>, api=OpenCL, #spirv.resource_limits<>>} {
+gpu.module @test_kernel   {
   gpu.func @transpose(%a: memref<512x8xf32>, %b: memref<8x512xf32>)  kernel attributes {VectorComputeFunctionINTEL, spirv.entry_point_abi = #spirv.entry_point_abi<>} {
 
     %c32 = arith.constant 32 : index
@@ -355,4 +350,100 @@ gpu.module @kernel  attributes {spirv.target_env = #spirv.target_env<#spirv.vce<
     xegpu.store_nd %2, %3: vector<8x32xf32>, !xegpu.tensor_desc<8x32xf32, #t>
     gpu.return
   }
-}
\ No newline at end of file
+}
+
+// -----
+gpu.module @test_kernel {
+  // CHECK-LABEL: test_prefetch_load_store_update
+  // CHECK-SAME: [[arg0:%.+]]: ui64
+  // CHECK-COUNT-2: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+  // CHECK-COUNT-2: xegpu.prefetch {{.*}} : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+   // CHECK-COUNT-2: xegpu.update_offset {{.*}} : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xindex>
+   // CHECK-COUNT-2: xegpu.load  {{.*}} : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
+  // CHECK-COUNT-2: xegpu.store  {{.*}} : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1>
+
+  gpu.func @test_prefetch_load_store_update(%src: ui64)  {
+
+    %cst = arith.constant dense<[
+    0,   8,  16,  24,  32,  40,  48,  56,
+    64,  72,  80,  88,  96, 104, 112, 120,
+    128, 136, 144, 152, 160, 168, 176, 184,
+    192, 200, 208, 216, 224, 232, 240, 248 
+    ]> : vector<32xindex>
+
+    %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
+    xegpu.prefetch %tdesc: !xegpu.tensor_desc<32xf32,  #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>
+   
+    %delta = arith.constant dense<[
+    32,   32,  32,  32,  32,  32,  32,  32,
+    32,   32,  32,  32,  32,  32,  32,  64,
+    128, 128, 128, 128, 128, 128, 128, 128,
+    128, 128, 128, 128, 128, 128, 128, 256 
+    ]> : vector<32xindex>
+    %new_tdesc = xegpu.update_offset %tdesc, %delta
+              : !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xindex>     
+ 
+    %c17 = arith.constant 17: index
+    %mask = vector.create_mask %c17: vector<32xi1>
+
+    %ld_vec = xegpu.load %new_tdesc, %mask: !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, vector<32xi1> -> vector<32xf32>
+
+    %st_vec = arith.addf %ld_vec, %ld_vec : vector<32xf32>
+    xegpu.store %st_vec, %tdesc, %mask: 
+                 vector<32xf32>, 
+                 !xegpu.tensor_desc<32xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<inst_data = [16]>>, 
+                 vector<32xi1>
+  
+    gpu.return
+  }
+  
+}
+
+// -----
+
+gpu.module @test_kernel   {
+  // CHECK-LABEL: test_prefetch_load_store_update_chunk
+  // CHECK-SAME: [[arg0:%.+]]: ui64
+  // CHECK-COUNT-4: xegpu.create_tdesc [[arg0]], {{.*}} : ui64, vector<16xindex> -> !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+  // CHECK-COUNT-4: xegpu.prefetch {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>
+   // CHECK-COUNT-4: xegpu.update_offset {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xindex>
+   // CHECK-COUNT-4: xegpu.load  {{.*}} : !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1> -> vector<2x16xf32>
+  // CHECK-COUNT-4: xegpu.store  {{.*}} : vector<2x16xf32>, !xegpu.tensor_desc<16x2xf32, #xegpu.scatter_tdesc_attr<chunk_size = 2 : i64>>, vector<16xi1>
+
+  gpu.func @test_prefetch_load_store_update_chunk(%src: ui64)  {
+
+    %cst = arith.constant dense<[
+      0,   8,  16,  24,  32,  40,  48,  56,
+      64,  72,  80,  88,  96, 104, 112, 120,
+      128, 136, 144, 152, 160, 168, 176, 184,
+      192, 200, 208, 216, 224, 232, 240, 248 
+    ]> : vector<32xindex>
+
+    %tdesc = xegpu.create_tdesc %src, %cst : ui64, vector<32xindex> -> !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
+    xegpu.prefetch %tdesc: !xegpu.tensor_desc<32x4xf32,  #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>
+   
+    %delta = arith.constant dense<[
+      32,   32,  32,  32,  32,  32,  32,  32,
+      32,   32,  32,  32,  32,  32,  32,  64,
+      128, 128, 128, 128, 128, 128, 128, 128,
+      128, 128, 128, 128, 128, 128, 128, 256 
+    ]> : vector<32xindex>
+    %new_tdesc = xegpu.update_offset %tdesc, %delta
+              : !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xindex>     
+ 
+    %c17 = arith.constant 17: index
+    %mask = vector.create_mask %c17: vector<32xi1>
+
+    %ld_vec = xegpu.load %new_tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, vector<32xi1> -> vector<4x32xf32>
+
+    %st_vec = arith.addf %ld_vec, %ld_vec : vector<4x32xf32>
+    xegpu.store %st_vec, %tdesc, %mask <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>, transpose}>: 
+                 vector<4x32xf32>, 
+                 !xegpu.tensor_desc<32x4xf32, #xegpu.scatter_tdesc_attr<chunk_size=4>, #xegpu.layout<inst_data = [16, 2]>>, 
+                 vector<32xi1>
+  
+    gpu.return
+  }
+}
+
+
diff --git a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
index 4400d6d9625f..c84eb7419854 100644
--- a/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
+++ b/mlir/test/lib/Dialect/XeGPU/TestXeGPUTransforms.cpp
@@ -102,14 +102,14 @@ struct TestXeGPUUnrollingPatterns
           // attribute
           if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(type)) {
             Attribute encoding = tdescTy.getEncoding();
-            auto layout = llvm::dyn_cast_if_present<xegpu::LayoutAttr>(
-                tdescTy.getLayout());
+            auto layout = tdescTy.getLayoutAttr();
 
             // If the encoding is a ScatterTensorDescAttr, we need to
             // potentially adjust the chunk size based on the inst_data.
-            if (encoding && mlir::isa<xegpu::ScatterTensorDescAttr>(encoding)) {
+            if (tdescTy.isScattered()) {
               auto scatterAttr =
-                  mlir::dyn_cast<xegpu::ScatterTensorDescAttr>(encoding);
+                  llvm::dyn_cast_if_present<xegpu::ScatterTensorDescAttr>(
+                      encoding);
               int64_t chunkSize = scatterAttr.getChunkSize().getInt();
 
               if (chunkSize > 1) {
@@ -118,12 +118,10 @@ struct TestXeGPUUnrollingPatterns
                 if (!instData.empty())
                   blockedChunkSize = instData.asArrayRef().back();
 
-                auto chunkSizeAttr = mlir::IntegerAttr::get(
-                    mlir::IntegerType::get(ctx, 64), blockedChunkSize);
-
                 // To create a new attribute with a different chunk_size:
                 auto newEncoding = xegpu::ScatterTensorDescAttr::get(
-                    ctx, scatterAttr.getMemorySpace(), chunkSizeAttr);
+                    ctx, scatterAttr.getMemorySpace().getValue(),
+                    blockedChunkSize);
 
                 encoding = newEncoding;
               }
-- 
cgit v1.2.3


From 7b6963ea672f8fedbbaefd15eaca943495709d37 Mon Sep 17 00:00:00 2001
From: Muhammad Omair Javaid <omair.javaid@linaro.org>
Date: Thu, 19 Jun 2025 03:06:46 +0500
Subject: [compiler-rt] [Fuzzer] Fix tests linking buildbot failure (#144495)

Fix for #144495 by 6f4add3 broke sanitizer-aarch64-linux buildbot.

compiler-rt/lib/fuzzer/tests build failed because the linker was
looking gcc_s without '-l' appended.

The CMake script was adding the library name without the required
'-l' prefix. This patch adds the -l prefix changing gcc_s to -lgcc_s
and gcc to -lgcc.

https://lab.llvm.org/buildbot/#/builders/51/builds/18170
---
 compiler-rt/lib/fuzzer/tests/CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
index 543f486a9d50..c5885ccccd20 100644
--- a/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
+++ b/compiler-rt/lib/fuzzer/tests/CMakeLists.txt
@@ -45,10 +45,10 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
     list(APPEND FUZZER_UNWINDER_LIBS ${COMPILER_RT_UNWINDER_LINK_LIBS})
   elseif(COMPILER_RT_HAS_GCC_S_LIB)
     # As a fallback, use the shared libgcc_s library.
-    list(APPEND FUZZER_UNWINDER_LIBS gcc_s)
+    list(APPEND FUZZER_UNWINDER_LIBS -lgcc_s)
   elseif(COMPILER_RT_HAS_GCC_LIB)
     # As a final fallback, use the static libgcc library.
-    list(APPEND FUZZER_UNWINDER_LIBS gcc)
+    list(APPEND FUZZER_UNWINDER_LIBS -lgcc)
   elseif(NOT COMPILER_RT_USE_BUILTINS_LIBRARY)
     # If no unwinder is found and we aren't using the builtins library
     message(FATAL_ERROR "Fuzzer tests require a suitable unwinder, but none was found.")
-- 
cgit v1.2.3


From 00ae89a1cbece94412cf832e47fdf449a611ad24 Mon Sep 17 00:00:00 2001
From: zGoldthorpe <Zach.Goldthorpe@amd.com>
Date: Wed, 18 Jun 2025 16:35:01 -0600
Subject: Revert "[IPO] Added attributor for identifying invariant loads"
 (#144808)

Reverts llvm/llvm-project#141800

The implementation critically misunderstands the `AAMemoryBehavior`
attributor, which it relies on heavily.

@shiltian, since I do not have commit permissions.
---
 llvm/include/llvm/Transforms/IPO/Attributor.h      |  41 ---
 llvm/lib/Transforms/IPO/Attributor.cpp             |   2 -
 llvm/lib/Transforms/IPO/AttributorAttributes.cpp   | 339 ------------------
 .../Attributor/AMDGPU/tag-invariant-loads.ll       | 382 ---------------------
 .../Transforms/Attributor/dereferenceable-1.ll     |   1 +
 .../Attributor/value-simplify-local-remote.ll      |  22 +-
 6 files changed, 15 insertions(+), 772 deletions(-)
 delete mode 100644 llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll

diff --git a/llvm/include/llvm/Transforms/IPO/Attributor.h b/llvm/include/llvm/Transforms/IPO/Attributor.h
index f19f3292c479..e6eb756df987 100644
--- a/llvm/include/llvm/Transforms/IPO/Attributor.h
+++ b/llvm/include/llvm/Transforms/IPO/Attributor.h
@@ -6335,47 +6335,6 @@ struct AAUnderlyingObjects : AbstractAttribute {
                           AA::ValueScope Scope = AA::Interprocedural) const = 0;
 };
 
-/// An abstract interface for identifying pointers from which loads can be
-/// marked invariant.
-struct AAInvariantLoadPointer : public AbstractAttribute {
-  AAInvariantLoadPointer(const IRPosition &IRP) : AbstractAttribute(IRP) {}
-
-  /// See AbstractAttribute::isValidIRPositionForInit
-  static bool isValidIRPositionForInit(Attributor &A, const IRPosition &IRP) {
-    if (!IRP.getAssociatedType()->isPointerTy())
-      return false;
-
-    return AbstractAttribute::isValidIRPositionForInit(A, IRP);
-  }
-
-  /// Create an abstract attribute view for the position \p IRP.
-  static AAInvariantLoadPointer &createForPosition(const IRPosition &IRP,
-                                                   Attributor &A);
-
-  /// Return true if the pointer's contents are known to remain invariant.
-  virtual bool isKnownInvariant() const = 0;
-  virtual bool isKnownLocallyInvariant() const = 0;
-
-  /// Return true if the pointer's contents are assumed to remain invariant.
-  virtual bool isAssumedInvariant() const = 0;
-  virtual bool isAssumedLocallyInvariant() const = 0;
-
-  /// See AbstractAttribute::getName().
-  StringRef getName() const override { return "AAInvariantLoadPointer"; }
-
-  /// See AbstractAttribute::getIdAddr().
-  const char *getIdAddr() const override { return &ID; }
-
-  /// This function should return true if the type of the \p AA is
-  /// AAInvariantLoadPointer
-  static bool classof(const AbstractAttribute *AA) {
-    return (AA->getIdAddr() == &ID);
-  }
-
-  /// Unique ID (due to the unique address).
-  static const char ID;
-};
-
 /// An abstract interface for address space information.
 struct AAAddressSpace : public StateWrapper<BooleanState, AbstractAttribute> {
   AAAddressSpace(const IRPosition &IRP, Attributor &A)
diff --git a/llvm/lib/Transforms/IPO/Attributor.cpp b/llvm/lib/Transforms/IPO/Attributor.cpp
index a2548258ddaf..dac1f7a30c37 100644
--- a/llvm/lib/Transforms/IPO/Attributor.cpp
+++ b/llvm/lib/Transforms/IPO/Attributor.cpp
@@ -3612,8 +3612,6 @@ void Attributor::identifyDefaultAbstractAttributes(Function &F) {
       if (SimplifyAllLoads)
         getAssumedSimplified(IRPosition::value(I), nullptr,
                              UsedAssumedInformation, AA::Intraprocedural);
-      getOrCreateAAFor<AAInvariantLoadPointer>(
-          IRPosition::value(*LI->getPointerOperand()));
       getOrCreateAAFor<AAAddressSpace>(
           IRPosition::value(*LI->getPointerOperand()));
     } else {
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
index 5cb8f888354b..3799a696f67a 100644
--- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
+++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp
@@ -191,7 +191,6 @@ PIPE_OPERATOR(AAInterFnReachability)
 PIPE_OPERATOR(AAPointerInfo)
 PIPE_OPERATOR(AAAssumptionInfo)
 PIPE_OPERATOR(AAUnderlyingObjects)
-PIPE_OPERATOR(AAInvariantLoadPointer)
 PIPE_OPERATOR(AAAddressSpace)
 PIPE_OPERATOR(AAAllocationInfo)
 PIPE_OPERATOR(AAIndirectCallInfo)
@@ -12534,342 +12533,6 @@ private:
 };
 } // namespace
 
-/// --------------------- Invariant Load Pointer -------------------------------
-namespace {
-
-struct AAInvariantLoadPointerImpl
-    : public StateWrapper<BitIntegerState<uint8_t, 15>,
-                          AAInvariantLoadPointer> {
-
-  enum {
-    // pointer does not alias within the bounds of the function
-    IS_NOALIAS = 1 << 0,
-    // pointer is not involved in any effectful instructions within the bounds
-    // of the function
-    IS_NOEFFECT = 1 << 1,
-    // loads are invariant within the bounds of the function
-    IS_LOCALLY_INVARIANT = 1 << 2,
-    // memory lifetime is constrained within the bounds of the function
-    IS_LOCALLY_CONSTRAINED = 1 << 3,
-
-    IS_BEST_STATE = IS_NOALIAS | IS_NOEFFECT | IS_LOCALLY_INVARIANT |
-                    IS_LOCALLY_CONSTRAINED,
-  };
-  static_assert(getBestState() == IS_BEST_STATE, "Unexpected best state");
-
-  using Base =
-      StateWrapper<BitIntegerState<uint8_t, 15>, AAInvariantLoadPointer>;
-
-  // the BitIntegerState is optimistic about IS_NOALIAS and IS_NOEFFECT, but
-  // pessimistic about IS_KNOWN_INVARIANT
-  AAInvariantLoadPointerImpl(const IRPosition &IRP, Attributor &A)
-      : Base(IRP) {}
-
-  bool isKnownInvariant() const final {
-    return isKnownLocallyInvariant() && isKnown(IS_LOCALLY_CONSTRAINED);
-  }
-
-  bool isKnownLocallyInvariant() const final {
-    if (isKnown(IS_LOCALLY_INVARIANT))
-      return true;
-    return isKnown(IS_NOALIAS | IS_NOEFFECT);
-  }
-
-  bool isAssumedInvariant() const final {
-    return isAssumedLocallyInvariant() && isAssumed(IS_LOCALLY_CONSTRAINED);
-  }
-
-  bool isAssumedLocallyInvariant() const final {
-    if (isAssumed(IS_LOCALLY_INVARIANT))
-      return true;
-    return isAssumed(IS_NOALIAS | IS_NOEFFECT);
-  }
-
-  ChangeStatus updateImpl(Attributor &A) override {
-    ChangeStatus Changed = ChangeStatus::UNCHANGED;
-
-    Changed |= updateNoAlias(A);
-    if (requiresNoAlias() && !isAssumed(IS_NOALIAS))
-      return indicatePessimisticFixpoint();
-
-    Changed |= updateNoEffect(A);
-
-    Changed |= updateLocalInvariance(A);
-
-    return Changed;
-  }
-
-  ChangeStatus manifest(Attributor &A) override {
-    if (!isKnownInvariant())
-      return ChangeStatus::UNCHANGED;
-
-    ChangeStatus Changed = ChangeStatus::UNCHANGED;
-    const Value *Ptr = &getAssociatedValue();
-    const auto TagInvariantLoads = [&](const Use &U, bool &) {
-      if (U.get() != Ptr)
-        return true;
-      auto *I = dyn_cast<Instruction>(U.getUser());
-      if (!I)
-        return true;
-
-      // Ensure that we are only changing uses from the corresponding callgraph
-      // SSC in the case that the AA isn't run on the entire module
-      if (!A.isRunOn(I->getFunction()))
-        return true;
-
-      if (I->hasMetadata(LLVMContext::MD_invariant_load))
-        return true;
-
-      if (auto *LI = dyn_cast<LoadInst>(I)) {
-        LI->setMetadata(LLVMContext::MD_invariant_load,
-                        MDNode::get(LI->getContext(), {}));
-        Changed = ChangeStatus::CHANGED;
-      }
-      return true;
-    };
-
-    (void)A.checkForAllUses(TagInvariantLoads, *this, *Ptr);
-    return Changed;
-  }
-
-  /// See AbstractAttribute::getAsStr().
-  const std::string getAsStr(Attributor *) const override {
-    if (isKnownInvariant())
-      return "load-invariant pointer";
-    return "non-invariant pointer";
-  }
-
-  /// See AbstractAttribute::trackStatistics().
-  void trackStatistics() const override {}
-
-private:
-  /// Indicate that noalias is required for the pointer to be invariant.
-  bool requiresNoAlias() const {
-    switch (getPositionKind()) {
-    default:
-      // Conservatively default to require noalias.
-      return true;
-    case IRP_FLOAT:
-    case IRP_RETURNED:
-    case IRP_CALL_SITE:
-      return false;
-    case IRP_CALL_SITE_RETURNED: {
-      const auto &CB = cast<CallBase>(getAnchorValue());
-      return !isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
-          &CB, /*MustPreserveNullness=*/false);
-    }
-    case IRP_ARGUMENT: {
-      const Function *F = getAssociatedFunction();
-      assert(F && "no associated function for argument");
-      return !isCallableCC(F->getCallingConv());
-    }
-    }
-  }
-
-  bool isExternal() const {
-    const Function *F = getAssociatedFunction();
-    if (!F)
-      return true;
-    return isCallableCC(F->getCallingConv()) &&
-           getPositionKind() != IRP_CALL_SITE_RETURNED;
-  }
-
-  ChangeStatus updateNoAlias(Attributor &A) {
-    if (isKnown(IS_NOALIAS) || !isAssumed(IS_NOALIAS))
-      return ChangeStatus::UNCHANGED;
-
-    // Try to use AANoAlias.
-    if (const auto *ANoAlias = A.getOrCreateAAFor<AANoAlias>(
-            getIRPosition(), this, DepClassTy::REQUIRED)) {
-      if (ANoAlias->isKnownNoAlias()) {
-        addKnownBits(IS_NOALIAS);
-        return ChangeStatus::CHANGED;
-      }
-
-      if (!ANoAlias->isAssumedNoAlias()) {
-        removeAssumedBits(IS_NOALIAS);
-        return ChangeStatus::CHANGED;
-      }
-
-      return ChangeStatus::UNCHANGED;
-    }
-
-    // Try to infer noalias from argument attribute, since it is applicable for
-    // the duration of the function.
-    if (const Argument *Arg = getAssociatedArgument()) {
-      if (Arg->hasNoAliasAttr()) {
-        addKnownBits(IS_NOALIAS);
-        return ChangeStatus::UNCHANGED;
-      }
-
-      // Noalias information is not provided, and cannot be inferred,
-      // so we conservatively assume the pointer aliases.
-      removeAssumedBits(IS_NOALIAS);
-      return ChangeStatus::CHANGED;
-    }
-
-    return ChangeStatus::UNCHANGED;
-  }
-
-  ChangeStatus updateNoEffect(Attributor &A) {
-    if (isKnown(IS_NOEFFECT) || !isAssumed(IS_NOEFFECT))
-      return ChangeStatus::UNCHANGED;
-
-    if (!getAssociatedFunction())
-      return indicatePessimisticFixpoint();
-
-    const auto HasNoEffectLoads = [&](const Use &U, bool &) {
-      const auto *LI = dyn_cast<LoadInst>(U.getUser());
-      return !LI || !LI->mayHaveSideEffects();
-    };
-    if (!A.checkForAllUses(HasNoEffectLoads, *this, getAssociatedValue()))
-      return indicatePessimisticFixpoint();
-
-    // Try to use AAMemoryBehavior to infer readonly attribute.
-    if (const auto *AMemoryBehavior = A.getOrCreateAAFor<AAMemoryBehavior>(
-            getIRPosition(), this, DepClassTy::REQUIRED)) {
-      if (!AMemoryBehavior->isAssumedReadOnly())
-        return indicatePessimisticFixpoint();
-
-      if (AMemoryBehavior->isKnownReadOnly()) {
-        addKnownBits(IS_NOEFFECT);
-        return ChangeStatus::UNCHANGED;
-      }
-
-      return ChangeStatus::UNCHANGED;
-    }
-
-    if (const Argument *Arg = getAssociatedArgument()) {
-      if (Arg->onlyReadsMemory()) {
-        addKnownBits(IS_NOEFFECT);
-        return ChangeStatus::UNCHANGED;
-      }
-
-      // Readonly information is not provided, and cannot be inferred from
-      // AAMemoryBehavior.
-      return indicatePessimisticFixpoint();
-    }
-
-    return ChangeStatus::UNCHANGED;
-  }
-
-  ChangeStatus updateLocalInvariance(Attributor &A) {
-    if (isKnown(IS_LOCALLY_INVARIANT) || !isAssumed(IS_LOCALLY_INVARIANT))
-      return ChangeStatus::UNCHANGED;
-
-    // try to infer invariance from underlying objects
-    const auto *AUO = A.getOrCreateAAFor<AAUnderlyingObjects>(
-        getIRPosition(), this, DepClassTy::REQUIRED);
-    if (!AUO)
-      return ChangeStatus::UNCHANGED;
-
-    bool UsedAssumedInformation = false;
-    const auto IsLocallyInvariantLoadIfPointer = [&](const Value &V) {
-      if (!V.getType()->isPointerTy())
-        return true;
-      const auto *IsInvariantLoadPointer =
-          A.getOrCreateAAFor<AAInvariantLoadPointer>(IRPosition::value(V), this,
-                                                     DepClassTy::REQUIRED);
-      // Conservatively fail if invariance cannot be inferred.
-      if (!IsInvariantLoadPointer)
-        return false;
-
-      if (IsInvariantLoadPointer->isKnownLocallyInvariant())
-        return true;
-      if (!IsInvariantLoadPointer->isAssumedLocallyInvariant())
-        return false;
-
-      UsedAssumedInformation = true;
-      return true;
-    };
-    if (!AUO->forallUnderlyingObjects(IsLocallyInvariantLoadIfPointer))
-      return indicatePessimisticFixpoint();
-
-    if (const auto *CB = dyn_cast<CallBase>(&getAnchorValue())) {
-      if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
-              CB, /*MustPreserveNullness=*/false)) {
-        for (const Value *Arg : CB->args()) {
-          if (!IsLocallyInvariantLoadIfPointer(*Arg))
-            return indicatePessimisticFixpoint();
-        }
-      }
-    }
-
-    if (!UsedAssumedInformation) {
-      // Pointer is known and not just assumed to be locally invariant.
-      addKnownBits(IS_LOCALLY_INVARIANT);
-      return ChangeStatus::CHANGED;
-    }
-
-    return ChangeStatus::UNCHANGED;
-  }
-};
-
-struct AAInvariantLoadPointerFloating final : AAInvariantLoadPointerImpl {
-  AAInvariantLoadPointerFloating(const IRPosition &IRP, Attributor &A)
-      : AAInvariantLoadPointerImpl(IRP, A) {}
-};
-
-struct AAInvariantLoadPointerReturned final : AAInvariantLoadPointerImpl {
-  AAInvariantLoadPointerReturned(const IRPosition &IRP, Attributor &A)
-      : AAInvariantLoadPointerImpl(IRP, A) {}
-
-  void initialize(Attributor &) override {
-    removeAssumedBits(IS_LOCALLY_CONSTRAINED);
-  }
-};
-
-struct AAInvariantLoadPointerCallSiteReturned final
-    : AAInvariantLoadPointerImpl {
-  AAInvariantLoadPointerCallSiteReturned(const IRPosition &IRP, Attributor &A)
-      : AAInvariantLoadPointerImpl(IRP, A) {}
-
-  void initialize(Attributor &A) override {
-    const Function *F = getAssociatedFunction();
-    assert(F && "no associated function for return from call");
-
-    if (!F->isDeclaration() && !F->isIntrinsic())
-      return AAInvariantLoadPointerImpl::initialize(A);
-
-    const auto &CB = cast<CallBase>(getAnchorValue());
-    if (isIntrinsicReturningPointerAliasingArgumentWithoutCapturing(
-            &CB, /*MustPreserveNullness=*/false))
-      return AAInvariantLoadPointerImpl::initialize(A);
-
-    if (F->onlyReadsMemory() && F->hasNoSync())
-      return AAInvariantLoadPointerImpl::initialize(A);
-
-    // At this point, the function is opaque, so we conservatively assume
-    // non-invariance.
-    indicatePessimisticFixpoint();
-  }
-};
-
-struct AAInvariantLoadPointerArgument final : AAInvariantLoadPointerImpl {
-  AAInvariantLoadPointerArgument(const IRPosition &IRP, Attributor &A)
-      : AAInvariantLoadPointerImpl(IRP, A) {}
-
-  void initialize(Attributor &) override {
-    const Function *F = getAssociatedFunction();
-    assert(F && "no associated function for argument");
-
-    if (!isCallableCC(F->getCallingConv())) {
-      addKnownBits(IS_LOCALLY_CONSTRAINED);
-      return;
-    }
-
-    if (!F->hasLocalLinkage())
-      removeAssumedBits(IS_LOCALLY_CONSTRAINED);
-  }
-};
-
-struct AAInvariantLoadPointerCallSiteArgument final
-    : AAInvariantLoadPointerImpl {
-  AAInvariantLoadPointerCallSiteArgument(const IRPosition &IRP, Attributor &A)
-      : AAInvariantLoadPointerImpl(IRP, A) {}
-};
-} // namespace
-
 /// ------------------------ Address Space  ------------------------------------
 namespace {
 
@@ -13375,7 +13038,6 @@ const char AAInterFnReachability::ID = 0;
 const char AAPointerInfo::ID = 0;
 const char AAAssumptionInfo::ID = 0;
 const char AAUnderlyingObjects::ID = 0;
-const char AAInvariantLoadPointer::ID = 0;
 const char AAAddressSpace::ID = 0;
 const char AAAllocationInfo::ID = 0;
 const char AAIndirectCallInfo::ID = 0;
@@ -13510,7 +13172,6 @@ CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPotentialValues)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoUndef)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AANoFPClass)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAPointerInfo)
-CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAInvariantLoadPointer)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAddressSpace)
 CREATE_VALUE_ABSTRACT_ATTRIBUTE_FOR_POSITION(AAAllocationInfo)
 
diff --git a/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll b/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
deleted file mode 100644
index ace68a19bf41..000000000000
--- a/llvm/test/Transforms/Attributor/AMDGPU/tag-invariant-loads.ll
+++ /dev/null
@@ -1,382 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=attributor %s -S | FileCheck %s --check-prefix=AMDGCN
-
-@G = addrspace(1) global i32 zeroinitializer, align 4
-declare void @clobber(i32) #0
-declare ptr addrspace(1) @get_ptr() #0
-declare noalias ptr addrspace(1) @get_noalias_ptr() #0
-declare noalias ptr addrspace(1) @get_untouched_ptr() #1
-
-define void @test_nonkernel(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define void @test_nonkernel(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2:[0-9]+]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6:[0-9]+]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; may not be !invariant.load, as the caller may modify %ptr
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_plain(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_plain(
-; AMDGCN-SAME: ptr addrspace(1) nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; may not be !invariant.load, as %ptr may alias a pointer in @clobber
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_noalias_ptr(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_ptr(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0:![0-9]+]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_gep(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_gep(
-; AMDGCN-SAME: ptr addrspace(1) nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %val = load i32, ptr addrspace(1) %gep, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_noalias_gep(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_noalias_gep(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly align 4 captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i32 4
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[GEP]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %gep = getelementptr i32, ptr addrspace(1) %ptr, i32 4
-  %val = load i32, ptr addrspace(1) %gep, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_swap(ptr addrspace(1) noalias %ptr, i32 inreg %swap) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_swap(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]], i32 inreg [[SWAP:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    store i32 [[SWAP]], ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; cannot be !invariant.load due to the write to %ptr
-  store i32 %swap, ptr addrspace(1) %ptr, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_volatile(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_volatile(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef align 4 [[PTR:%.*]]) #[[ATTR3:[0-9]+]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load volatile i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load volatile i32, ptr addrspace(1) %ptr, align 4
-  ;; volatiles loads cannot be !invariant.load
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_unordered(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_unordered(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] unordered, align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load atomic i32, ptr addrspace(1) %ptr unordered, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_monotonic(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_monotonic(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load atomic i32, ptr addrspace(1) [[PTR]] monotonic, align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load atomic i32, ptr addrspace(1) %ptr monotonic, align 4
-  ;; atomic loads with ordering guarantees may have side effects
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_global() {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_global(
-; AMDGCN-SAME: ) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) @G, align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = load i32, ptr addrspace(1) @G, align 4
-  ;; is not an !invariant.load as global variables may change
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define internal i32 @test_internal_noalias_load(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define internal i32 @test_internal_noalias_load(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR4:[0-9]+]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    ret i32 [[VAL]]
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; is an !invariant.load due to its only caller @test_call_internal_noalias
-  ret i32 %val
-}
-
-define amdgpu_kernel void @test_call_internal_noalias(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_noalias(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_noalias_load(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR7:[0-9]+]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = call i32 @test_internal_noalias_load(ptr addrspace(1) %ptr)
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define internal i32 @test_internal_load(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define internal i32 @test_internal_load(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR4]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    ret i32 [[VAL]]
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; may not be an !invariant.load since the pointer in @test_call_internal may alias
-  ret i32 %val
-}
-
-define amdgpu_kernel void @test_call_internal(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal(
-; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_load(ptr addrspace(1) nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR7]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = call i32 @test_internal_load(ptr addrspace(1) %ptr)
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define internal i32 @test_internal_written(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define internal i32 @test_internal_written(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) dereferenceable_or_null(4) [[PTR:%.*]]) #[[ATTR4]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    ret i32 [[VAL]]
-;
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; cannot be an !invariant.load because of the write in caller @test_call_internal_written
-  ret i32 %val
-}
-
-define amdgpu_kernel void @test_call_internal_written(ptr addrspace(1) noalias %ptr, i32 inreg %x) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_internal_written(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree captures(none) [[PTR:%.*]], i32 inreg [[X:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[VAL:%.*]] = call i32 @test_internal_written(ptr addrspace(1) noalias nofree noundef readonly align 4 captures(none) [[PTR]]) #[[ATTR7]]
-; AMDGCN-NEXT:    store i32 [[X]], ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %val = call i32 @test_internal_written(ptr addrspace(1) %ptr)
-  store i32 %x, ptr addrspace(1) %ptr
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_call_ptr() {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_ptr(
-; AMDGCN-SAME: ) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_ptr() #[[ATTR6]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %ptr = call ptr addrspace(1) @get_ptr()
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; may not be an !invariant.load since %ptr may alias
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_call_noalias_ptr() {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_noalias_ptr(
-; AMDGCN-SAME: ) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[PTR:%.*]] = call align 4 ptr addrspace(1) @get_noalias_ptr() #[[ATTR6]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %ptr = call ptr addrspace(1) @get_noalias_ptr()
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; may not be an !invariant.load since %ptr may have been written to before returning
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_call_untouched_ptr() {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_call_untouched_ptr(
-; AMDGCN-SAME: ) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[PTR:%.*]] = call noalias align 4 ptr addrspace(1) @get_untouched_ptr() #[[ATTR8:[0-9]+]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %ptr = call ptr addrspace(1) @get_untouched_ptr()
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_make_buffer(ptr addrspace(1) %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer(
-; AMDGCN-SAME: ptr addrspace(1) nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR9:[0-9]+]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
-  %val = load i32, ptr addrspace(7) %rsrc, align 4
-  ;; original %ptr may alias
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_make_buffer_noalias(ptr addrspace(1) noalias %ptr) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_make_buffer_noalias(
-; AMDGCN-SAME: ptr addrspace(1) noalias nofree readonly captures(none) [[PTR:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[RSRC:%.*]] = call align 4 ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) [[PTR]], i16 noundef 0, i32 noundef 0, i32 noundef 0) #[[ATTR9]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(7) [[RSRC]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %rsrc = call ptr addrspace(7) @llvm.amdgcn.make.buffer.rsrc.p7.p1(ptr addrspace(1) %ptr, i16 0, i32 0, i32 0)
-  %val = load i32, ptr addrspace(7) %rsrc, align 4
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_selected_load(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load(
-; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; either pointer yields an !invariant.load
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_selected_load_partial_noalias(i1 inreg %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_selected_load_partial_noalias(
-; AMDGCN-SAME: i1 inreg [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:    [[PTR:%.*]] = select i1 [[COND]], ptr addrspace(1) [[PTR_TRUE]], ptr addrspace(1) [[PTR_FALSE]]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-  %ptr = select i1 %cond, ptr addrspace(1) %ptr.true, ptr addrspace(1) %ptr.false
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; %ptr.false may alias, so no !invariant.load
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_branch_load(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) noalias %ptr.false) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load(
-; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:  [[ENTRY:.*:]]
-; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
-; AMDGCN:       [[TRUE]]:
-; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR6]]
-; AMDGCN-NEXT:    br label %[[FINISH:.*]]
-; AMDGCN:       [[FALSE]]:
-; AMDGCN-NEXT:    br label %[[FINISH]]
-; AMDGCN:       [[FINISH]]:
-; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4, !invariant.load [[META0]]
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-entry:
-  br i1 %cond, label %true, label %false
-true:
-  call void @clobber(i32 1)
-  br label %finish
-false:
-  br label %finish
-finish:
-  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; either pointer yields an !invariant.load
-  call void @clobber(i32 %val)
-  ret void
-}
-
-define amdgpu_kernel void @test_branch_load_partial_noalias(i1 %cond, ptr addrspace(1) noalias %ptr.true, ptr addrspace(1) %ptr.false) {
-; AMDGCN-LABEL: define amdgpu_kernel void @test_branch_load_partial_noalias(
-; AMDGCN-SAME: i1 noundef [[COND:%.*]], ptr addrspace(1) noalias nofree readonly captures(none) [[PTR_TRUE:%.*]], ptr addrspace(1) nofree readonly captures(none) [[PTR_FALSE:%.*]]) #[[ATTR2]] {
-; AMDGCN-NEXT:  [[ENTRY:.*:]]
-; AMDGCN-NEXT:    br i1 [[COND]], label %[[TRUE:.*]], label %[[FALSE:.*]]
-; AMDGCN:       [[TRUE]]:
-; AMDGCN-NEXT:    call void @clobber(i32 noundef 1) #[[ATTR6]]
-; AMDGCN-NEXT:    br label %[[FINISH:.*]]
-; AMDGCN:       [[FALSE]]:
-; AMDGCN-NEXT:    br label %[[FINISH]]
-; AMDGCN:       [[FINISH]]:
-; AMDGCN-NEXT:    [[PTR:%.*]] = phi ptr addrspace(1) [ [[PTR_TRUE]], %[[TRUE]] ], [ [[PTR_FALSE]], %[[FALSE]] ]
-; AMDGCN-NEXT:    [[VAL:%.*]] = load i32, ptr addrspace(1) [[PTR]], align 4
-; AMDGCN-NEXT:    call void @clobber(i32 [[VAL]]) #[[ATTR6]]
-; AMDGCN-NEXT:    ret void
-;
-entry:
-  br i1 %cond, label %true, label %false
-true:
-  call void @clobber(i32 1)
-  br label %finish
-false:
-  br label %finish
-finish:
-  %ptr = phi ptr addrspace(1) [ %ptr.true, %true ], [ %ptr.false, %false ]
-  %val = load i32, ptr addrspace(1) %ptr, align 4
-  ;; ptr.false may alias, so no !invariant.load
-  call void @clobber(i32 %val)
-  ret void
-}
-
-attributes #0 = { nofree norecurse nosync nounwind willreturn }
-attributes #1 = { nofree norecurse nosync nounwind willreturn readonly }
-;.
-; AMDGCN: [[META0]] = !{}
-;.
diff --git a/llvm/test/Transforms/Attributor/dereferenceable-1.ll b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
index 5bff2a2e6b20..07e2d5ea1575 100644
--- a/llvm/test/Transforms/Attributor/dereferenceable-1.ll
+++ b/llvm/test/Transforms/Attributor/dereferenceable-1.ll
@@ -207,6 +207,7 @@ define void @f7_1(ptr %ptr, i1 %cnd) {
 ; CHECK-LABEL: define {{[^@]+}}@f7_1
 ; CHECK-SAME: (ptr noundef nonnull align 4 dereferenceable(4) [[PTR:%.*]], i1 noundef [[CND:%.*]]) #[[ATTR2]] {
 ; CHECK-NEXT:    [[A:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
+; CHECK-NEXT:    [[PTR_0:%.*]] = load i32, ptr [[PTR]], align 4
 ; CHECK-NEXT:    [[B:%.*]] = tail call i32 @unkown_f(ptr noundef nonnull align 4 dereferenceable(4) [[PTR]]) #[[ATTR1]]
 ; CHECK-NEXT:    br i1 [[CND]], label [[IF_TRUE:%.*]], label [[IF_FALSE:%.*]]
 ; CHECK:       if.true:
diff --git a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
index 4767244800d2..374d5ba7ff52 100644
--- a/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
+++ b/llvm/test/Transforms/Attributor/value-simplify-local-remote.ll
@@ -135,7 +135,7 @@ define internal %S @foo.1(ptr %foo.this) {
 ; TUNIT-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[FOO_THIS]], ptr [[FOO_THIS]], align 8
 ; TUNIT-NEXT:    call void @bar.2(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR5:[0-9]+]]
-; TUNIT-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8, !invariant.load [[META8:![0-9]+]]
+; TUNIT-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; TUNIT-NEXT:    ret [[S]] [[FOO_RET]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
@@ -145,7 +145,7 @@ define internal %S @foo.1(ptr %foo.this) {
 ; CGSCC-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[FOO_THIS]], ptr [[FOO_THIS]], align 8
 ; CGSCC-NEXT:    call void @bar.2(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[FOO_THIS]]) #[[ATTR6]]
-; CGSCC-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8, !invariant.load [[META8:![0-9]+]]
+; CGSCC-NEXT:    [[FOO_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[FOO_RET]]
 ;
 entry:
@@ -234,7 +234,7 @@ define internal %S @bar.5(ptr %this) {
 ; TUNIT-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; TUNIT-NEXT:    call void @baz.6(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR4]]
-; TUNIT-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8, !invariant.load [[META8]]
+; TUNIT-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; TUNIT-NEXT:    ret [[S]] [[BAR_RET]]
 ;
 ; CGSCC: Function Attrs: mustprogress nofree nosync nounwind willreturn memory(argmem: readwrite)
@@ -244,7 +244,7 @@ define internal %S @bar.5(ptr %this) {
 ; CGSCC-NEXT:    [[RETVAL:%.*]] = alloca [[S:%.*]], i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; CGSCC-NEXT:    call void @baz.6(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull align 8 dereferenceable(8) [[THIS]]) #[[ATTR9:[0-9]+]]
-; CGSCC-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8, !invariant.load [[META8]]
+; CGSCC-NEXT:    [[BAR_RET:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[BAR_RET]]
 ;
 entry:
@@ -286,7 +286,7 @@ define internal void @boom(ptr %this, ptr %data) {
 ; TUNIT-NEXT:  entry:
 ; TUNIT-NEXT:    [[DATA_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; TUNIT-NEXT:    store ptr [[DATA]], ptr [[DATA_ADDR]], align 8
-; TUNIT-NEXT:    [[V:%.*]] = load ptr, ptr [[DATA_ADDR]], align 8, !invariant.load [[META8]]
+; TUNIT-NEXT:    [[V:%.*]] = load ptr, ptr [[DATA_ADDR]], align 8
 ; TUNIT-NEXT:    store ptr [[V]], ptr [[THIS]], align 8
 ; TUNIT-NEXT:    ret void
 ;
@@ -342,6 +342,14 @@ define %S.2 @t3.helper() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[RETVAL:%.*]] = alloca [[S_2:%.*]], align 8
 ; CHECK-NEXT:    call void @ext1(ptr noundef nonnull align 8 dereferenceable(24) [[RETVAL]])
+; CHECK-NEXT:    [[DOTFCA_0_LOAD:%.*]] = load ptr, ptr [[RETVAL]], align 8
+; CHECK-NEXT:    [[DOTFCA_0_INSERT:%.*]] = insertvalue [[S_2]] poison, ptr [[DOTFCA_0_LOAD]], 0
+; CHECK-NEXT:    [[DOTFCA_1_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 1
+; CHECK-NEXT:    [[DOTFCA_1_LOAD:%.*]] = load i64, ptr [[DOTFCA_1_GEP]], align 8
+; CHECK-NEXT:    [[DOTFCA_1_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_0_INSERT]], i64 [[DOTFCA_1_LOAD]], 1
+; CHECK-NEXT:    [[DOTFCA_2_GEP:%.*]] = getelementptr inbounds [[S_2]], ptr [[RETVAL]], i32 0, i32 2
+; CHECK-NEXT:    [[DOTFCA_2_LOAD:%.*]] = load i64, ptr [[DOTFCA_2_GEP]], align 8
+; CHECK-NEXT:    [[DOTFCA_2_INSERT:%.*]] = insertvalue [[S_2]] [[DOTFCA_1_INSERT]], i64 [[DOTFCA_2_LOAD]], 2
 ; CHECK-NEXT:    ret [[S_2]] zeroinitializer
 ;
 entry:
@@ -500,7 +508,7 @@ define internal %S @t4a(ptr %this) {
 ; CGSCC-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, i32 0, align 8
 ; CGSCC-NEXT:    store ptr [[THIS]], ptr [[THIS]], align 8
 ; CGSCC-NEXT:    call void @t4b(ptr noalias nofree noundef nonnull writeonly align 8 captures(none) dereferenceable(8) [[RETVAL]], ptr nofree noundef nonnull writeonly align 8 dereferenceable(8) [[THIS]]) #[[ATTR6]]
-; CGSCC-NEXT:    [[TMP0:%.*]] = load [[S]], ptr [[RETVAL]], align 8, !invariant.load [[META8]]
+; CGSCC-NEXT:    [[TMP0:%.*]] = load [[S]], ptr [[RETVAL]], align 8
 ; CGSCC-NEXT:    ret [[S]] [[TMP0]]
 ;
 entry:
@@ -615,7 +623,6 @@ entry:
 ; TUNIT: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
 ; TUNIT: [[META6:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 2}
 ; TUNIT: [[META7:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
-; TUNIT: [[META8]] = !{}
 ;.
 ; CGSCC: [[META0:![0-9]+]] = !{i32 2, !"SDK Version", [2 x i32] [i32 11, i32 5]}
 ; CGSCC: [[META1:![0-9]+]] = !{i32 1, !"wchar_size", i32 4}
@@ -625,5 +632,4 @@ entry:
 ; CGSCC: [[META5:![0-9]+]] = !{i32 7, !"frame-pointer", i32 2}
 ; CGSCC: [[META6:![0-9]+]] = !{i32 7, !"Dwarf Version", i32 2}
 ; CGSCC: [[META7:![0-9]+]] = !{i32 2, !"Debug Info Version", i32 3}
-; CGSCC: [[META8]] = !{}
 ;.
-- 
cgit v1.2.3


From e0933ab5ae4856c4aa188a5ea16716b3a8d0840b Mon Sep 17 00:00:00 2001
From: Chelsea Cassanova <chelsea_cassanova@apple.com>
Date: Wed, 18 Jun 2025 15:39:25 -0700
Subject: Revert "[lldb][target] Add progress report for wait-attaching to
 process" (#144810)

This is breaking TestCreateAfterAttach.py on Ubuntu:

```
======================================================================
FAIL: test_create_after_attach_dwo (TestCreateAfterAttach.CreateAfterAttachTestCase.test_create_after_attach_dwo)
   Test thread creation after process attach.
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/buildbot/worker/as-builder-9/lldb-remote-linux-ubuntu/llvm-project/lldb/packages/Python/lldbsuite/test/lldbtest.py", line 1804, in test_method
    return attrvalue(self)
           ^^^^^^^^^^^^^^^
  File "/home/buildbot/worker/as-builder-9/lldb-remote-linux-ubuntu/llvm-project/lldb/packages/Python/lldbsuite/test/decorators.py", line 149, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/home/buildbot/worker/as-builder-9/lldb-remote-linux-ubuntu/llvm-project/lldb/test/API/functionalities/thread/create_after_attach/TestCreateAfterAttach.py", line 36, in test_create_after_attach
    self.runCmd("process attach -p " + str(pid))
  File "/home/buildbot/worker/as-builder-9/lldb-remote-linux-ubuntu/llvm-project/lldb/packages/Python/lldbsuite/test/lldbtest.py", line 1005, in runCmd
    self.assertTrue(self.res.Succeeded(), msg + output)
AssertionError: False is not true : Command 'process attach -p 1474309' did not return successfully
Error output:
error: attach failed: lost connection
```

on the buildbots for lldb-remote-linux-ubuntu, lldb-arm-ubuntu,
lldb-aarch64-ubuntu, lldb-arm-ubuntu.
---
 lldb/source/Target/Target.cpp                      |  1 -
 .../progress_reporting/TestProgressReporting.py    | 31 ----------------------
 2 files changed, 32 deletions(-)

diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp
index 8f8d2ef21cc5..45a9e1196a04 100644
--- a/lldb/source/Target/Target.cpp
+++ b/lldb/source/Target/Target.cpp
@@ -3546,7 +3546,6 @@ llvm::Expected<TraceSP> Target::GetTraceOrCreate() {
 }
 
 Status Target::Attach(ProcessAttachInfo &attach_info, Stream *stream) {
-  Progress attach_progress("Waiting to attach to process");
   m_stats.SetLaunchOrAttachTime();
   auto state = eStateInvalid;
   auto process_sp = GetProcessSP();
diff --git a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
index 8198c50a5ff0..9af53845ca1b 100644
--- a/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
+++ b/lldb/test/API/functionalities/progress_reporting/TestProgressReporting.py
@@ -2,7 +2,6 @@
 Test that we are able to broadcast and receive progress events from lldb
 """
 import lldb
-import threading
 
 import lldbsuite.test.lldbutil as lldbutil
 
@@ -17,36 +16,6 @@ class TestProgressReporting(TestBase):
             self.broadcaster, lldb.SBDebugger.eBroadcastBitProgress
         )
 
-    def test_wait_attach_progress_reporting(self):
-        """Test that progress reports for wait attaching work as intended."""
-        self.build()
-        target = self.dbg.CreateTarget(None)
-
-        # Wait attach to a process, then check to see that a progress report was created
-        # and that its message is correct for waiting to attach to a process.
-        class AttachThread(threading.Thread):
-            def __init__(self, target):
-                threading.Thread.__init__(self)
-                self.target = target
-
-            def run(self):
-                self.target.AttachToProcessWithName(
-                    lldb.SBListener(), "a.out", True, lldb.SBError()
-                )
-
-        thread = AttachThread(target)
-        thread.start()
-
-        event = lldbutil.fetch_next_event(self, self.listener, self.broadcaster)
-        progress_data = lldb.SBDebugger.GetProgressDataFromEvent(event)
-        message = progress_data.GetValueForKey("message").GetStringValue(100)
-        self.assertEqual(message, "Waiting to attach to process")
-
-        # Interrupt the process attach to keep the test from stalling.
-        target.process.SendAsyncInterrupt()
-
-        thread.join()
-
     def test_dwarf_symbol_loading_progress_report(self):
         """Test that we are able to fetch dwarf symbol loading progress events"""
         self.build()
-- 
cgit v1.2.3


From 780c0ef7fb97027aa21c2ee6b02282693f908a20 Mon Sep 17 00:00:00 2001
From: Andrew Rogers <andrurogerz@gmail.com>
Date: Wed, 18 Jun 2025 15:41:13 -0700
Subject: [llvm] explicitly link llvm-jitlink-executor with ExecutionEngine
 (#144778)

## Overview
Explicitly link `llvm-jitlink-executor` with `ExecutionEngine` to avoid
link failures when building LLVM as a Windows DLL. This link dependency
should probably have always been declared here, but didn't matter when
building against an LLVM static library because it was (presumably)
picked up as a transitive dependency.

This change is required to enable the Windows DLL build because
`llvm-jitlink-executor` is declared using `add_llvm_utility` which
invokes `add_llvm_executable` with
[`DISABLE_LLVM_LINK_LLVM_DYLIB`](https://github.com/llvm/llvm-project/blob/main/llvm/cmake/modules/AddLLVM.cmake#L500-L502)
so it links statically against its dependencies instead of against the
main LLVM library.

## Background
The effort to support building LLVM as a Windows DLL is tracked in
#109483. Additional context is provided in [this
discourse](https://discourse.llvm.org/t/psa-annotating-llvm-public-interface/85307).
---
 llvm/tools/llvm-jitlink/llvm-jitlink-executor/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/CMakeLists.txt b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/CMakeLists.txt
index f2daa294eec0..792ecf544f61 100644
--- a/llvm/tools/llvm-jitlink/llvm-jitlink-executor/CMakeLists.txt
+++ b/llvm/tools/llvm-jitlink/llvm-jitlink-executor/CMakeLists.txt
@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  ExecutionEngine
   OrcShared
   OrcTargetProcess
   Support
-- 
cgit v1.2.3


From bb1f5c3189c4d8d30e3b1273e0b774a7ccdbd86a Mon Sep 17 00:00:00 2001
From: Guy David <49722543+guy-david@users.noreply.github.com>
Date: Thu, 19 Jun 2025 01:53:36 +0300
Subject: [AArch64] Lower jump table cases threshold to 10 (#143632)

Previous stabs at this setting
(https://github.com/llvm/llvm-project/pull/71166) hypertuned it for
SPEC2017, but Clang's own compilation can benefit from a slightly lower
threshold, yielding a 0.3% improvement in compile time, while still not
regressing SPEC.

Most notable beneficiaries of this change are:
 - `llvm::Instruction::getNumSuccessors` (11 cases)
 - `llvm::Instruction::getSuccessor` (11 cases)

Test Suite with a bootstrapped build:
```
Tests: 4316
Metric: compile_time

Program                                       compile_time
                                              lhs          rhs    diff
SingleSour...ce/UnitTests/SignlessTypes/div     0.02         0.02  3.0%
SingleSour.../UnitTests/SignlessTypes/cast2     0.02         0.02  2.8%
SingleSource/Benchmarks/Misc/flops-4            0.02         0.02  1.9%
SingleSour...ebra/solvers/cholesky/cholesky     0.05         0.05  1.8%
SingleSour...tTests/2020-01-06-coverage-006     0.02         0.02  1.7%
SingleSour...ce/Benchmarks/Stanford/FloatMM     0.03         0.03  1.7%
SingleSour...9-04-16-BitfieldInitialization     0.02         0.02  1.7%
SingleSour...nitTests/2003-07-08-BitOpsTest     0.02         0.02  1.7%
MultiSourc...marks/Prolangs-C++/vcirc/vcirc     0.02         0.02  1.6%
MultiSourc...Prolangs-C/fixoutput/fixoutput     0.05         0.05  1.5%
SingleSour...h/stencils/jacobi-1d/jacobi-1d     0.04         0.04  1.4%
MultiSourc...rks/Prolangs-C++/office/office     0.28         0.28  1.4%
SingleSour...arks/Adobe-C++/functionobjects     0.39         0.40  1.3%
SingleSour...Tests/2003-10-29-ScalarReplBug     0.02         0.02  1.2%
SingleSour...arks/Adobe-C++/stepanov_vector     0.41         0.42  1.2%
                           Geomean difference                     -0.3%
      compile_time
l/r            lhs          rhs        diff
count  4316.000000  4316.000000  469.000000
mean   0.057747     0.057595    -0.003034
std    0.544528     0.543139     0.007625
min    0.000000     0.000000    -0.035294
25%    0.000000     0.000000    -0.007006
50%    0.000000     0.000000    -0.003257
75%    0.000000     0.000000     0.000000
max    18.295300    18.252500    0.030151
```
---
 llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 2 +-
 llvm/test/CodeGen/AArch64/min-jump-table.ll  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index a28e6bad0dca..68ed10570a52 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -73,7 +73,7 @@ static cl::opt<AArch64PAuth::AuthCheckMethod>
                                cl::values(AUTH_CHECK_METHOD_CL_VALUES_LR));
 
 static cl::opt<unsigned> AArch64MinimumJumpTableEntries(
-    "aarch64-min-jump-table-entries", cl::init(13), cl::Hidden,
+    "aarch64-min-jump-table-entries", cl::init(10), cl::Hidden,
     cl::desc("Set minimum number of entries to use a jump table on AArch64"));
 
 static cl::opt<unsigned> AArch64StreamingHazardSize(
diff --git a/llvm/test/CodeGen/AArch64/min-jump-table.ll b/llvm/test/CodeGen/AArch64/min-jump-table.ll
index 98b89210f5a0..7cdff6e435f7 100644
--- a/llvm/test/CodeGen/AArch64/min-jump-table.ll
+++ b/llvm/test/CodeGen/AArch64/min-jump-table.ll
@@ -105,7 +105,7 @@ entry:
 ; CHECK4-NEXT: Jump Tables:
 ; CHECK8-NEXT: Jump Tables:
 ; CHECK12-NEXT: Jump Tables:
-; CHECK-DEFAULT-NOT: {{^}}Jump Tables:
+; CHECK-DEFAULT: {{^}}Jump Tables:
 
 bb1: tail call void @ext(i32 1, i32 0) br label %return
 bb2: tail call void @ext(i32 2, i32 2) br label %return
-- 
cgit v1.2.3


From 5f69d680e2cc94dcb30a7f29e8144725530a6da4 Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Wed, 18 Jun 2025 19:30:43 -0400
Subject: Revert "[HLSL][SPIRV] Add vk::constant_id attribute." (#144812)

Reverts llvm/llvm-project#143544
---
 clang/include/clang/Basic/Attr.td                  |   8 -
 clang/include/clang/Basic/AttrDocs.td              |  15 --
 clang/include/clang/Basic/Builtins.td              |  13 --
 clang/include/clang/Basic/DiagnosticSemaKinds.td   |   4 -
 clang/include/clang/Sema/SemaHLSL.h                |   5 +-
 clang/lib/CodeGen/CGHLSLBuiltins.cpp               |  74 --------
 clang/lib/CodeGen/CodeGenFunction.h                |   6 -
 clang/lib/Sema/SemaDecl.cpp                        |  13 --
 clang/lib/Sema/SemaDeclAttr.cpp                    |   3 -
 clang/lib/Sema/SemaHLSL.cpp                        | 120 +-----------
 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl    | 130 -------------
 .../inline-spirv/SpirvType.alignment.hlsl          |  16 ++
 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl |  68 +++++++
 .../vk-features/SpirvType.alignment.hlsl           |  16 --
 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl  |  68 -------
 .../CodeGenHLSL/vk-features/vk.spec-constant.hlsl  | 210 ---------------------
 clang/test/SemaHLSL/vk.spec-constant.error.hlsl    |  37 ----
 17 files changed, 86 insertions(+), 720 deletions(-)
 delete mode 100644 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
 create mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
 create mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
 delete mode 100644 clang/test/SemaHLSL/vk.spec-constant.error.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 27fea7dea0a5..f113cd2ba2fb 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -5023,14 +5023,6 @@ def HLSLVkExtBuiltinInput : InheritableAttr {
   let Documentation = [HLSLVkExtBuiltinInputDocs];
 }
 
-def HLSLVkConstantId : InheritableAttr {
-  let Spellings = [CXX11<"vk", "constant_id">];
-  let Args = [IntArgument<"Id">];
-  let Subjects = SubjectList<[ExternalGlobalVar]>;
-  let LangOpts = [HLSL];
-  let Documentation = [VkConstantIdDocs];
-}
-
 def RandomizeLayout : InheritableAttr {
   let Spellings = [GCC<"randomize_layout">];
   let Subjects = SubjectList<[Record]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 43442f177ab7..6051e1fc4511 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -8252,21 +8252,6 @@ and https://microsoft.github.io/hlsl-specs/proposals/0013-wave-size-range.html
   }];
 }
 
-def VkConstantIdDocs : Documentation {
-  let Category = DocCatFunction;
-  let Content = [{
-The ``vk::constant_id`` attribute specifies the id for a SPIR-V specialization
-constant. The attribute applies to const global scalar variables. The variable must be initialized with a C++11 constexpr.
-In SPIR-V, the
-variable will be replaced with an `OpSpecConstant` with the given id.
-The syntax is:
-
-.. code-block:: text
-
-  ``[[vk::constant_id(<Id>)]] const T Name = <Init>``
-}];
-}
-
 def RootSignatureDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index d65b3a5d2f44..68cd3d790e78 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5065,19 +5065,6 @@ def HLSLGroupMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void()";
 }
 
-class HLSLScalarTemplate
-    : Template<["bool", "char", "short", "int", "long long int",
-                "unsigned short", "unsigned int", "unsigned long long int",
-                "__fp16", "float", "double"],
-               ["_bool", "_char", "_short", "_int", "_longlong", "_ushort",
-                "_uint", "_ulonglong", "_half", "_float", "_double"]>;
-
-def HLSLGetSpirvSpecConstant : LangBuiltin<"HLSL_LANG">, HLSLScalarTemplate {
-  let Spellings = ["__builtin_get_spirv_spec_constant"];
-  let Attributes = [NoThrow, Const, Pure];
-  let Prototype = "T(unsigned int, T)";
-}
-
 // Builtins for XRay.
 def XRayCustomEvent : Builtin {
   let Spellings = ["__xray_customevent"];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 34b798a09c21..979ff60b73b7 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12927,10 +12927,6 @@ def err_spirv_enum_not_int : Error<
 def err_spirv_enum_not_valid : Error<
    "invalid value for %select{storage class}0 argument">;
 
-def err_specialization_const
-    : Error<"variable with 'vk::constant_id' attribute must be a const "
-            "int/float/enum/bool and be initialized with a literal">;
-
 // errors of expect.with.probability
 def err_probability_not_constant_float : Error<
    "probability argument to __builtin_expect_with_probability must be constant "
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 97091792ba23..33c4b8d1568b 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -98,8 +98,6 @@ public:
   HLSLWaveSizeAttr *mergeWaveSizeAttr(Decl *D, const AttributeCommonInfo &AL,
                                       int Min, int Max, int Preferred,
                                       int SpelledArgsCount);
-  HLSLVkConstantIdAttr *
-  mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL, int Id);
   HLSLShaderAttr *mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                                   llvm::Triple::EnvironmentType ShaderType);
   HLSLParamModifierAttr *
@@ -137,7 +135,6 @@ public:
   void handleRootSignatureAttr(Decl *D, const ParsedAttr &AL);
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
   void handleWaveSizeAttr(Decl *D, const ParsedAttr &AL);
-  void handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL);
@@ -174,7 +171,7 @@ public:
   QualType getInoutParameterType(QualType Ty);
 
   bool transformInitList(const InitializedEntity &Entity, InitListExpr *Init);
-  bool handleInitialization(VarDecl *VDecl, Expr *&Init);
+
   void deduceAddressSpace(VarDecl *Decl);
 
 private:
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index cbc5ef9cb0d5..ccf45c0c6ff1 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -12,7 +12,6 @@
 
 #include "CGBuiltin.h"
 #include "CGHLSLRuntime.h"
-#include "CodeGenFunction.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -215,43 +214,6 @@ static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,
   }
 }
 
-// Returns the mangled name for a builtin function that the SPIR-V backend
-// will expand into a spec Constant.
-static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
-                                               ASTContext &Context) {
-  // The parameter types for our conceptual intrinsic function.
-  QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
-
-  // Create a temporary FunctionDecl for the builtin fuction. It won't be
-  // added to the AST.
-  FunctionProtoType::ExtProtoInfo EPI;
-  QualType FnType =
-      Context.getFunctionType(SpecConstantType, ClangParamTypes, EPI);
-  DeclarationName FuncName = &Context.Idents.get("__spirv_SpecConstant");
-  FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
-      Context, Context.getTranslationUnitDecl(), SourceLocation(),
-      SourceLocation(), FuncName, FnType, /*TSI=*/nullptr, SC_Extern);
-
-  // Attach the created parameter declarations to the function declaration.
-  SmallVector<ParmVarDecl *, 2> ParamDecls;
-  for (QualType ParamType : ClangParamTypes) {
-    ParmVarDecl *PD = ParmVarDecl::Create(
-        Context, FnDeclForMangling, SourceLocation(), SourceLocation(),
-        /*IdentifierInfo*/ nullptr, ParamType, /*TSI*/ nullptr, SC_None,
-        /*DefaultArg*/ nullptr);
-    ParamDecls.push_back(PD);
-  }
-  FnDeclForMangling->setParams(ParamDecls);
-
-  // Get the mangled name.
-  std::string Name;
-  llvm::raw_string_ostream MangledNameStream(Name);
-  MangleContext *Mangler = Context.createMangleContext();
-  Mangler->mangleName(FnDeclForMangling, MangledNameStream);
-  MangledNameStream.flush();
-  return Name;
-}
-
 Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
                                             const CallExpr *E,
                                             ReturnValueSlot ReturnValue) {
@@ -811,42 +773,6 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return EmitRuntimeCall(
         Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
   }
-  case Builtin::BI__builtin_get_spirv_spec_constant_bool:
-  case Builtin::BI__builtin_get_spirv_spec_constant_short:
-  case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
-  case Builtin::BI__builtin_get_spirv_spec_constant_int:
-  case Builtin::BI__builtin_get_spirv_spec_constant_uint:
-  case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
-  case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
-  case Builtin::BI__builtin_get_spirv_spec_constant_half:
-  case Builtin::BI__builtin_get_spirv_spec_constant_float:
-  case Builtin::BI__builtin_get_spirv_spec_constant_double: {
-    llvm::Function *SpecConstantFn = getSpecConstantFunction(E->getType());
-    llvm::Value *SpecId = EmitScalarExpr(E->getArg(0));
-    llvm::Value *DefaultVal = EmitScalarExpr(E->getArg(1));
-    llvm::Value *Args[] = {SpecId, DefaultVal};
-    return Builder.CreateCall(SpecConstantFn, Args);
-  }
   }
   return nullptr;
 }
-
-llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
-    const clang::QualType &SpecConstantType) {
-
-  // Find or create the declaration for the function.
-  llvm::Module *M = &CGM.getModule();
-  std::string MangledName =
-      getSpecConstantFunctionName(SpecConstantType, getContext());
-  llvm::Function *SpecConstantFn = M->getFunction(MangledName);
-
-  if (!SpecConstantFn) {
-    llvm::Type *IntType = ConvertType(getContext().IntTy);
-    llvm::Type *RetTy = ConvertType(SpecConstantType);
-    llvm::Type *ArgTypes[] = {IntType, RetTy};
-    llvm::FunctionType *FnTy = llvm::FunctionType::get(RetTy, ArgTypes, false);
-    SpecConstantFn = llvm::Function::Create(
-        FnTy, llvm::GlobalValue::ExternalLinkage, MangledName, M);
-  }
-  return SpecConstantFn;
-}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 59f14b3e35fd..a5ab9df01dba 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4850,12 +4850,6 @@ public:
   llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
                                    ReturnValueSlot ReturnValue);
-
-  // Returns a builtin function that the SPIR-V backend will expand into a spec
-  // constant.
-  llvm::Function *
-  getSpecConstantFunction(const clang::QualType &SpecConstantType);
-
   llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index e1cccf068b5a..1bf72e5bb7b9 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2890,8 +2890,6 @@ static bool mergeDeclAttribute(Sema &S, NamedDecl *D,
     NewAttr = S.HLSL().mergeWaveSizeAttr(D, *WS, WS->getMin(), WS->getMax(),
                                          WS->getPreferred(),
                                          WS->getSpelledArgsCount());
-  else if (const auto *CI = dyn_cast<HLSLVkConstantIdAttr>(Attr))
-    NewAttr = S.HLSL().mergeVkConstantIdAttr(D, *CI, CI->getId());
   else if (const auto *SA = dyn_cast<HLSLShaderAttr>(Attr))
     NewAttr = S.HLSL().mergeShaderAttr(D, *SA, SA->getType());
   else if (isa<SuppressAttr>(Attr))
@@ -13759,10 +13757,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     return;
   }
 
-  if (getLangOpts().HLSL)
-    if (!HLSL().handleInitialization(VDecl, Init))
-      return;
-
   // Get the decls type and save a reference for later, since
   // CheckInitializerTypes may change it.
   QualType DclT = VDecl->getType(), SavT = DclT;
@@ -14185,13 +14179,6 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
       }
     }
 
-    // HLSL variable with the `vk::constant_id` attribute must be initialized.
-    if (!Var->isInvalidDecl() && Var->hasAttr<HLSLVkConstantIdAttr>()) {
-      Diag(Var->getLocation(), diag::err_specialization_const);
-      Var->setInvalidDecl();
-      return;
-    }
-
     if (!Var->isInvalidDecl() && RealDecl->hasAttr<LoaderUninitializedAttr>()) {
       if (Var->getStorageClass() == SC_Extern) {
         Diag(Var->getLocation(), diag::err_loader_uninitialized_extern_decl)
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index eba29e609cb0..1c2fa80e782d 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7590,9 +7590,6 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLVkExtBuiltinInput:
     S.HLSL().handleVkExtBuiltinInputAttr(D, AL);
     break;
-  case ParsedAttr::AT_HLSLVkConstantId:
-    S.HLSL().handleVkConstantIdAttr(D, AL);
-    break;
   case ParsedAttr::AT_HLSLSV_GroupThreadID:
     S.HLSL().handleSV_GroupThreadIDAttr(D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 9b43ee00810b..b55f4fd786b5 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -119,40 +119,6 @@ static ResourceClass getResourceClass(RegisterType RT) {
   llvm_unreachable("unexpected RegisterType value");
 }
 
-static Builtin::ID getSpecConstBuiltinId(QualType Type) {
-  const auto *BT = dyn_cast<BuiltinType>(Type);
-  if (!BT) {
-    if (!Type->isEnumeralType())
-      return Builtin::NotBuiltin;
-    return Builtin::BI__builtin_get_spirv_spec_constant_int;
-  }
-
-  switch (BT->getKind()) {
-  case BuiltinType::Bool:
-    return Builtin::BI__builtin_get_spirv_spec_constant_bool;
-  case BuiltinType::Short:
-    return Builtin::BI__builtin_get_spirv_spec_constant_short;
-  case BuiltinType::Int:
-    return Builtin::BI__builtin_get_spirv_spec_constant_int;
-  case BuiltinType::LongLong:
-    return Builtin::BI__builtin_get_spirv_spec_constant_longlong;
-  case BuiltinType::UShort:
-    return Builtin::BI__builtin_get_spirv_spec_constant_ushort;
-  case BuiltinType::UInt:
-    return Builtin::BI__builtin_get_spirv_spec_constant_uint;
-  case BuiltinType::ULongLong:
-    return Builtin::BI__builtin_get_spirv_spec_constant_ulonglong;
-  case BuiltinType::Half:
-    return Builtin::BI__builtin_get_spirv_spec_constant_half;
-  case BuiltinType::Float:
-    return Builtin::BI__builtin_get_spirv_spec_constant_float;
-  case BuiltinType::Double:
-    return Builtin::BI__builtin_get_spirv_spec_constant_double;
-  default:
-    return Builtin::NotBuiltin;
-  }
-}
-
 DeclBindingInfo *ResourceBindings::addDeclBindingInfo(const VarDecl *VD,
                                                       ResourceClass ResClass) {
   assert(getDeclBindingInfo(VD, ResClass) == nullptr &&
@@ -641,41 +607,6 @@ HLSLWaveSizeAttr *SemaHLSL::mergeWaveSizeAttr(Decl *D,
   return Result;
 }
 
-HLSLVkConstantIdAttr *
-SemaHLSL::mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL,
-                                int Id) {
-
-  auto &TargetInfo = getASTContext().getTargetInfo();
-  if (TargetInfo.getTriple().getArch() != llvm::Triple::spirv) {
-    Diag(AL.getLoc(), diag::warn_attribute_ignored) << AL;
-    return nullptr;
-  }
-
-  auto *VD = cast<VarDecl>(D);
-
-  if (getSpecConstBuiltinId(VD->getType()) == Builtin::NotBuiltin) {
-    Diag(VD->getLocation(), diag::err_specialization_const);
-    return nullptr;
-  }
-
-  if (!VD->getType().isConstQualified()) {
-    Diag(VD->getLocation(), diag::err_specialization_const);
-    return nullptr;
-  }
-
-  if (HLSLVkConstantIdAttr *CI = D->getAttr<HLSLVkConstantIdAttr>()) {
-    if (CI->getId() != Id) {
-      Diag(CI->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
-      Diag(AL.getLoc(), diag::note_conflicting_attribute);
-    }
-    return nullptr;
-  }
-
-  HLSLVkConstantIdAttr *Result =
-      ::new (getASTContext()) HLSLVkConstantIdAttr(getASTContext(), AL, Id);
-  return Result;
-}
-
 HLSLShaderAttr *
 SemaHLSL::mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                           llvm::Triple::EnvironmentType ShaderType) {
@@ -1226,15 +1157,6 @@ void SemaHLSL::handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL) {
                  HLSLVkExtBuiltinInputAttr(getASTContext(), AL, ID));
 }
 
-void SemaHLSL::handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL) {
-  uint32_t Id;
-  if (!SemaRef.checkUInt32Argument(AL, AL.getArgAsExpr(0), Id))
-    return;
-  HLSLVkConstantIdAttr *NewAttr = mergeVkConstantIdAttr(D, AL, Id);
-  if (NewAttr)
-    D->addAttr(NewAttr);
-}
-
 bool SemaHLSL::diagnoseInputIDType(QualType T, const ParsedAttr &AL) {
   const auto *VT = T->getAs<VectorType>();
 
@@ -3284,7 +3206,6 @@ static bool IsDefaultBufferConstantDecl(VarDecl *VD) {
   return VD->getDeclContext()->isTranslationUnit() &&
          QT.getAddressSpace() == LangAS::Default &&
          VD->getStorageClass() != SC_Static &&
-         !VD->hasAttr<HLSLVkConstantIdAttr>() &&
          !isInvalidConstantBufferLeafElementType(QT.getTypePtr());
 }
 
@@ -3352,8 +3273,7 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
     const Type *VarType = VD->getType().getTypePtr();
     while (VarType->isArrayType())
       VarType = VarType->getArrayElementTypeNoTypeQual();
-    if (VarType->isHLSLResourceRecord() ||
-        VD->hasAttr<HLSLVkConstantIdAttr>()) {
+    if (VarType->isHLSLResourceRecord()) {
       // Make the variable for resources static. The global externally visible
       // storage is accessed through the handle, which is a member. The variable
       // itself is not externally visible.
@@ -3776,41 +3696,3 @@ bool SemaHLSL::transformInitList(const InitializedEntity &Entity,
     Init->updateInit(Ctx, I, NewInit->getInit(I));
   return true;
 }
-
-bool SemaHLSL::handleInitialization(VarDecl *VDecl, Expr *&Init) {
-  const HLSLVkConstantIdAttr *ConstIdAttr =
-      VDecl->getAttr<HLSLVkConstantIdAttr>();
-  if (!ConstIdAttr)
-    return true;
-
-  ASTContext &Context = SemaRef.getASTContext();
-
-  APValue InitValue;
-  if (!Init->isCXX11ConstantExpr(Context, &InitValue)) {
-    Diag(VDecl->getLocation(), diag::err_specialization_const);
-    VDecl->setInvalidDecl();
-    return false;
-  }
-
-  Builtin::ID BID = getSpecConstBuiltinId(VDecl->getType());
-
-  // Argument 1: The ID from the attribute
-  int ConstantID = ConstIdAttr->getId();
-  llvm::APInt IDVal(Context.getIntWidth(Context.IntTy), ConstantID);
-  Expr *IdExpr = IntegerLiteral::Create(Context, IDVal, Context.IntTy,
-                                        ConstIdAttr->getLocation());
-
-  SmallVector<Expr *, 2> Args = {IdExpr, Init};
-  Expr *C = SemaRef.BuildBuiltinCallExpr(Init->getExprLoc(), BID, Args);
-  if (C->getType()->getCanonicalTypeUnqualified() !=
-      VDecl->getType()->getCanonicalTypeUnqualified()) {
-    C = SemaRef
-            .BuildCStyleCastExpr(SourceLocation(),
-                                 Context.getTrivialTypeSourceInfo(
-                                     Init->getType(), Init->getExprLoc()),
-                                 SourceLocation(), C)
-            .get();
-  }
-  Init = C;
-  return true;
-}
diff --git a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
deleted file mode 100644
index c0955c1ea7b4..000000000000
--- a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
+++ /dev/null
@@ -1,130 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute -x hlsl -ast-dump -o - %s | FileCheck %s
-
-// CHECK: VarDecl {{.*}} bool_const 'const hlsl_private bool' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'bool'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool (*)(unsigned int, bool) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'bool (unsigned int, bool) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_bool' 'bool (unsigned int, bool) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 1
-// CHECK-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' true
-[[vk::constant_id(1)]]
-const bool bool_const = true;
-
-// CHECK: VarDecl {{.*}} short_const 'const hlsl_private short' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'short'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short (*)(unsigned int, short) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'short (unsigned int, short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_short' 'short (unsigned int, short) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 2
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
-[[vk::constant_id(2)]]
-const short short_const = 4;
-
-// CHECK: VarDecl {{.*}} int_const 'const hlsl_private int' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'int'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
-[[vk::constant_id(3)]]
-const int int_const = 5;
-
-// CHECK: VarDecl {{.*}} long_const 'const hlsl_private long long' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'long long'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long (*)(unsigned int, long long) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'long long (unsigned int, long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_longlong' 'long long (unsigned int, long long) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
-[[vk::constant_id(4)]]
-const long long long_const = 8;
-
-// CHECK: VarDecl {{.*}} ushort_const 'const hlsl_private unsigned short' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'unsigned short'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short (*)(unsigned int, unsigned short) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned short (unsigned int, unsigned short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ushort' 'unsigned short (unsigned int, unsigned short) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10
-[[vk::constant_id(5)]]
-const unsigned short ushort_const = 10;
-
-// CHECK: VarDecl {{.*}} uint_const 'const hlsl_private unsigned int' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'unsigned int'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int (*)(unsigned int, unsigned int) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int (unsigned int, unsigned int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_uint' 'unsigned int (unsigned int, unsigned int) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 6
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12
-[[vk::constant_id(6)]]
-const unsigned int uint_const = 12;
-
-
-// CHECK: VarDecl {{.*}} ulong_const 'const hlsl_private unsigned long long' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'unsigned long long'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long (*)(unsigned int, unsigned long long) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned long long (unsigned int, unsigned long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ulonglong' 'unsigned long long (unsigned int, unsigned long long) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 7
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 25
-[[vk::constant_id(7)]]
-const unsigned long long ulong_const = 25;
-
-// CHECK: VarDecl {{.*}} half_const 'const hlsl_private half' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'half'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half (*)(unsigned int, half) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'half (unsigned int, half) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_half' 'half (unsigned int, half) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half' <FloatingCast>
-// CHECK-NEXT: FloatingLiteral {{.*}} 'float' 4.040000e+01
-[[vk::constant_id(8)]]
-const half half_const = 40.4;
-
-// CHECK: VarDecl {{.*}} float_const 'const hlsl_private float' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'float'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float (*)(unsigned int, float) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'float (unsigned int, float) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_float' 'float (unsigned int, float) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <IntegralToFloating>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 50
-[[vk::constant_id(8)]]
-const float float_const = 50;
-
-// CHECK: VarDecl {{.*}} double_const 'const hlsl_private double' static cinit
-// CHECK-NEXT: CallExpr {{.*}} 'double'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double (*)(unsigned int, double) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'double (unsigned int, double) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_double' 'double (unsigned int, double) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 9
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' <IntegralToFloating>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 100
-[[vk::constant_id(9)]]
-const double double_const = 100;
-
-// CHECK: VarDecl {{.*}} enum_const 'const hlsl_private E' static cinit
-// CHECK-NEXT: CStyleCastExpr {{.*}} 'E' <IntegralCast>
-// CHECK-NEXT: CallExpr {{.*}} 'int'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
-// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10 
-// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int' <IntegralCast>
-// CHECK-NEXT: DeclRefExpr {{.*}} 'E' EnumConstant {{.*}} 'e2' 'E' 
-enum E {
-    e0 = 10,
-    e1 = 20,
-    e2 = 30
-};
-
-[[vk::constant_id(10)]]
-const E enum_const = e2;
-
-// CHECK-NOT: CXXRecordDecl {{.*}} implicit struct __cblayout_$Globals definition
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
new file mode 100644
index 000000000000..41cdd7d21bcb
--- /dev/null
+++ b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
+
+// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
+struct S {
+    int a;
+    Int b;
+};
+
+[numthreads(1,1,1)]
+void main() {
+    S value;
+}
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
new file mode 100644
index 000000000000..7149be0122f4
--- /dev/null
+++ b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+template<class T, uint64_t Size>
+using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
+
+template<uint64_t Size>
+using ArrayBuffer = Array<RWBuffer<float>, Size>;
+
+typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
+
+typedef Array<Int, 5> ArrayInt;
+
+// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
+struct S {
+    ArrayBuffer<4> b;
+    Int i;
+};
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
+ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
+    return v;
+}
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
+Int getInt(Int v) {
+    return v;
+}
+
+// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
+// ArrayBuffer<4> g_buffers;
+// Int g_word;
+
+[numthreads(1, 1, 1)]
+void main() {
+    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    ArrayBuffer<4> buffers;
+
+    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
+    ArrayBuffer<0x123456789> longBuffers;
+
+    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+    Int word;
+
+    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
+    Int words[4];
+
+    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
+    ArrayInt words2;
+
+    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
+    S value;
+
+    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
+    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
+    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
+    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
+    Int word2 = getInt(word);
+}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
deleted file mode 100644
index 41cdd7d21bcb..000000000000
--- a/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
+++ /dev/null
@@ -1,16 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
-
-// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
-struct S {
-    int a;
-    Int b;
-};
-
-[numthreads(1,1,1)]
-void main() {
-    S value;
-}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
deleted file mode 100644
index 7149be0122f4..000000000000
--- a/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
+++ /dev/null
@@ -1,68 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-template<class T, uint64_t Size>
-using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
-
-template<uint64_t Size>
-using ArrayBuffer = Array<RWBuffer<float>, Size>;
-
-typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
-
-typedef Array<Int, 5> ArrayInt;
-
-// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
-struct S {
-    ArrayBuffer<4> b;
-    Int i;
-};
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
-ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
-    return v;
-}
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
-Int getInt(Int v) {
-    return v;
-}
-
-// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
-// ArrayBuffer<4> g_buffers;
-// Int g_word;
-
-[numthreads(1, 1, 1)]
-void main() {
-    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    ArrayBuffer<4> buffers;
-
-    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
-    ArrayBuffer<0x123456789> longBuffers;
-
-    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-    Int word;
-
-    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
-    Int words[4];
-
-    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
-    ArrayInt words2;
-
-    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
-    S value;
-
-    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
-    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
-    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
-    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
-    Int word2 = getInt(word);
-}
diff --git a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
deleted file mode 100644
index cbc1fa61eae2..000000000000
--- a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
+++ /dev/null
@@ -1,210 +0,0 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 5
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-[[vk::constant_id(1)]]
-const bool bool_const = true;
-
-[[vk::constant_id(1)]]
-const short short_const = 4;
-
-[[vk::constant_id(3)]]
-const int int_const = 5;
-
-[[vk::constant_id(4)]]
-const long long long_const = 8;
-
-[[vk::constant_id(5)]]
-const unsigned short ushort_const = 10;
-
-[[vk::constant_id(6)]]
-const unsigned int uint_const = 12;
-
-[[vk::constant_id(7)]]
-const unsigned long long ulong_const = 25;
-
-[[vk::constant_id(8)]]
-const half half_const = 40.4;
-
-[[vk::constant_id(8)]]
-const float float_const = 50.5;
-
-[[vk::constant_id(9)]]
-const double double_const = 100.2;
-
-enum E {
-    e0 = 10,
-    e1 = 20,
-    e2 = 30
-};
-
-[[vk::constant_id(10)]]
-const E enum_const = e2;
-
-[numthreads(1,1,1)]
-void main() {
-    bool b = bool_const;
-    short s = short_const;
-    int i = int_const;
-    long long l = long_const;
-    unsigned short us = ushort_const;
-    unsigned int ui = uint_const;
-    unsigned long long ul = ulong_const;
-    half h = half_const;
-    float f = float_const;
-    double d = double_const;
-    E e = enum_const;
-}
-//.
-// CHECK: @_ZL10bool_const = internal addrspace(10) global i32 0, align 4
-// CHECK: @_ZL11short_const = internal addrspace(10) global i16 0, align 2
-// CHECK: @_ZL9int_const = internal addrspace(10) global i32 0, align 4
-// CHECK: @_ZL10long_const = internal addrspace(10) global i64 0, align 8
-// CHECK: @_ZL12ushort_const = internal addrspace(10) global i16 0, align 2
-// CHECK: @_ZL10uint_const = internal addrspace(10) global i32 0, align 4
-// CHECK: @_ZL11ulong_const = internal addrspace(10) global i64 0, align 8
-// CHECK: @_ZL10half_const = internal addrspace(10) global float 0.000000e+00, align 4
-// CHECK: @_ZL11float_const = internal addrspace(10) global float 0.000000e+00, align 4
-// CHECK: @_ZL12double_const = internal addrspace(10) global double 0.000000e+00, align 8
-// CHECK: @_ZL10enum_const = internal addrspace(10) global i32 0, align 4
-//.
-// CHECK-LABEL: define internal spir_func void @_Z4mainv(
-// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[S:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[L:%.*]] = alloca i64, align 8
-// CHECK-NEXT:    [[US:%.*]] = alloca i16, align 2
-// CHECK-NEXT:    [[UI:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[UL:%.*]] = alloca i64, align 8
-// CHECK-NEXT:    [[H:%.*]] = alloca float, align 4
-// CHECK-NEXT:    [[F:%.*]] = alloca float, align 4
-// CHECK-NEXT:    [[D:%.*]] = alloca double, align 8
-// CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
-// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(10) @_ZL10bool_const, align 4
-// CHECK-NEXT:    [[LOADEDV:%.*]] = trunc i32 [[TMP1]] to i1
-// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[LOADEDV]] to i32
-// CHECK-NEXT:    store i32 [[STOREDV]], ptr [[B]], align 4
-// CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(10) @_ZL11short_const, align 2
-// CHECK-NEXT:    store i16 [[TMP2]], ptr [[S]], align 2
-// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(10) @_ZL9int_const, align 4
-// CHECK-NEXT:    store i32 [[TMP3]], ptr [[I]], align 4
-// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr addrspace(10) @_ZL10long_const, align 8
-// CHECK-NEXT:    store i64 [[TMP4]], ptr [[L]], align 8
-// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(10) @_ZL12ushort_const, align 2
-// CHECK-NEXT:    store i16 [[TMP5]], ptr [[US]], align 2
-// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(10) @_ZL10uint_const, align 4
-// CHECK-NEXT:    store i32 [[TMP6]], ptr [[UI]], align 4
-// CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr addrspace(10) @_ZL11ulong_const, align 8
-// CHECK-NEXT:    store i64 [[TMP7]], ptr [[UL]], align 8
-// CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr addrspace(10) @_ZL10half_const, align 4
-// CHECK-NEXT:    store float [[TMP8]], ptr [[H]], align 4
-// CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr addrspace(10) @_ZL11float_const, align 4
-// CHECK-NEXT:    store float [[TMP9]], ptr [[F]], align 4
-// CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr addrspace(10) @_ZL12double_const, align 8
-// CHECK-NEXT:    store double [[TMP10]], ptr [[D]], align 8
-// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr addrspace(10) @_ZL10enum_const, align 4
-// CHECK-NEXT:    store i32 [[TMP11]], ptr [[E]], align 4
-// CHECK-NEXT:    ret void
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init(
-// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i1 @_Z20__spirv_SpecConstantib(i32 1, i1 true)
-// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[TMP1]] to i32
-// CHECK-NEXT:    store i32 [[STOREDV]], ptr addrspace(10) @_ZL10bool_const, align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.1(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantis(i32 1, i16 4)
-// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL11short_const, align 2
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.2(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 3, i32 5)
-// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL9int_const, align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.3(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantix(i32 4, i64 8)
-// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL10long_const, align 8
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.4(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantit(i32 5, i16 10)
-// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL12ushort_const, align 2
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.5(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantij(i32 6, i32 12)
-// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10uint_const, align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.6(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantiy(i32 7, i64 25)
-// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL11ulong_const, align 8
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.7(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantiDh(i32 8, float 0x4044333340000000)
-// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL10half_const, align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.8(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantif(i32 8, float 5.050000e+01)
-// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL11float_const, align 4
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.9(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn double @_Z20__spirv_SpecConstantid(i32 9, double 0x40590CCCC0000000)
-// CHECK-NEXT:    store double [[TMP1]], ptr addrspace(10) @_ZL12double_const, align 8
-// CHECK-NEXT:    ret void
-//
-//
-// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.10(
-// CHECK-SAME: ) #[[ATTR3]] {
-// CHECK-NEXT:  [[ENTRY:.*:]]
-// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
-// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 10, i32 30)
-// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10enum_const, align 4
-// CHECK-NEXT:    ret void
diff --git a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
deleted file mode 100644
index 24873d272a54..000000000000
--- a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
+++ /dev/null
@@ -1,37 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan1.3-compute -verify %s
-// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.8-compute -verify %s
-
-#ifndef __spirv__
-// expected-warning@+2{{'constant_id' attribute ignored}}
-#endif
-[[vk::constant_id(0)]]
-const bool sc0 = true;
-
-#ifdef __spirv__
-// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
-[[vk::constant_id(1)]]
-const bool sc1 = sc0; // error
-
-// expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
-[[vk::constant_id(2)]]
-static const bool sc2 = false; // error
-
-// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
-[[vk::constant_id(3)]]
-const bool sc3; // error
-
-// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
-[[vk::constant_id(4)]]
-bool sc4 = false; // error
-
-// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
-[[vk::constant_id(5)]]
-const int2 sc5 = {0,0}; // error
-
-[numthreads(1,1,1)]
-void main() {
-  // expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
-  [[vk::constant_id(6)]]
-  const bool sc6 = false; // error
-}
-#endif
-- 
cgit v1.2.3


From d265105b8f50718a684d792d3ca957231d668533 Mon Sep 17 00:00:00 2001
From: David Justo <david.justo.1996@gmail.com>
Date: Wed, 18 Jun 2025 17:13:25 -0700
Subject: Augment `uncaught-exception.test` fuzzer test to be msvc-compatible
 (#125924)

Today, the `uncaught-exception.test` fuzzer test checks for the string
"libFuzzer: deadly signal" in the program output as the result of an
uncaught exception.

Although this is correct for `clang`, `msvc` reports a different error
message: "libFuzzer: uncaught C++ exception". Since `msvc` reuses the
`libFuzzer` infrastructure for ASan regression testing, it would help us
greatly if the test handled the `msvc` divergence more gracefully.

**This PR:** augments this test so check for a different string (namely
"libFuzzer: uncaught C++ exception") if the compiler target matches the
`msvc` naming scheme.

I understand if this is outside the scope of support for LLVM as well,
and I'm also open for different approaches here. Thanks!
---
 compiler-rt/test/fuzzer/uncaught-exception.test | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/compiler-rt/test/fuzzer/uncaught-exception.test b/compiler-rt/test/fuzzer/uncaught-exception.test
index b055c88f6d90..d1b98cfb7c74 100644
--- a/compiler-rt/test/fuzzer/uncaught-exception.test
+++ b/compiler-rt/test/fuzzer/uncaught-exception.test
@@ -4,7 +4,10 @@
 REQUIRES: windows
 RUN: %cpp_compiler %S/UncaughtException.cpp -o %t-UncaughtException
 
-RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s
+# Clang will fail the test with 'deadly signal', but other compilers may fail with different error messages.
+# For example, msvc fails with 'uncaught C++ exception'. So the error we check depends on the compiler target.
+RUN: not %run %t-UncaughtException 2>&1 | FileCheck %s --check-prefixes=CHECK-CRASH,%if target={{.*-windows-msvc.*}} %{CHECK-MSVC%} %else %{CHECK-ERROR%}
 
-CHECK: ERROR: libFuzzer: deadly signal
-CHECK: Test unit written to ./crash
+CHECK-ERROR: ERROR: libFuzzer: deadly signal
+CHECK-MSVC: ERROR: libFuzzer: uncaught C++ exception
+CHECK-CRASH: Test unit written to ./crash
-- 
cgit v1.2.3


From bc8908a4e93b0641e1c17f408885c8aebb308bbe Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 19 Jun 2025 09:36:10 +0900
Subject: ARM: Move declaration of supportSplitCSR to be public (#144679)

This is an implementation of a public method from the base
class, so it should also be public. Avoids unrelated diff
in a future patch.
---
 llvm/lib/Target/ARM/ARMISelLowering.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 87710ee29a24..357ca9ea5d20 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -716,6 +716,11 @@ class VectorType;
       return true;
     }
 
+    bool supportSplitCSR(MachineFunction *MF) const override {
+      return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
+             MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
+    }
+
     bool hasStandaloneRem(EVT VT) const override {
       return HasStandaloneRem;
     }
@@ -914,11 +919,6 @@ class VectorType;
                             SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
                             SDValue ThisVal, bool isCmseNSCall) const;
 
-    bool supportSplitCSR(MachineFunction *MF) const override {
-      return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
-          MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
-    }
-
     void initializeSplitCSR(MachineBasicBlock *Entry) const override;
     void insertCopiesSplitCSR(
       MachineBasicBlock *Entry,
-- 
cgit v1.2.3


From 874a02f05b6ebb4b5dbe0ab09beb9c3d5b36e237 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 19 Jun 2025 09:38:22 +0900
Subject: ARM: Move ABI helpers from Subtarget to TargetMachine (#144680)

These are module level concepts, and attaching them to the
function level subtarget is confusing. Similarly these other
helpers that only operate on the triple should also be removed
from the subtarget.
---
 llvm/lib/Target/ARM/ARMAsmPrinter.cpp    |  6 +++++-
 llvm/lib/Target/ARM/ARMAsmPrinter.h      |  2 ++
 llvm/lib/Target/ARM/ARMFastISel.cpp      | 15 ++++++++-------
 llvm/lib/Target/ARM/ARMISelLowering.cpp  | 27 +++++++++++++++++----------
 llvm/lib/Target/ARM/ARMISelLowering.h    |  3 +++
 llvm/lib/Target/ARM/ARMSubtarget.cpp     | 20 ++------------------
 llvm/lib/Target/ARM/ARMSubtarget.h       |  6 ------
 llvm/lib/Target/ARM/ARMTargetMachine.cpp | 16 ++++++++++++++++
 llvm/lib/Target/ARM/ARMTargetMachine.h   |  4 ++++
 9 files changed, 57 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
index 1443747709b7..2fd784373f4a 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -54,6 +54,10 @@ ARMAsmPrinter::ARMAsmPrinter(TargetMachine &TM,
     : AsmPrinter(TM, std::move(Streamer), ID), Subtarget(nullptr), AFI(nullptr),
       MCP(nullptr), InConstantPool(false), OptimizationGoals(-1) {}
 
+const ARMBaseTargetMachine &ARMAsmPrinter::getTM() const {
+  return static_cast<const ARMBaseTargetMachine &>(TM);
+}
+
 void ARMAsmPrinter::emitFunctionBodyEnd() {
   // Make sure to terminate any constant pools that were at the end
   // of the function.
@@ -750,7 +754,7 @@ void ARMAsmPrinter::emitAttributes() {
   ATS.emitAttribute(ARMBuildAttrs::ABI_align_preserved, 1);
 
   // Hard float.  Use both S and D registers and conform to AAPCS-VFP.
-  if (STI.isAAPCS_ABI() && TM.Options.FloatABIType == FloatABI::Hard)
+  if (getTM().isAAPCS_ABI() && TM.Options.FloatABIType == FloatABI::Hard)
     ATS.emitAttribute(ARMBuildAttrs::ABI_VFP_args, ARMBuildAttrs::HardFPAAPCS);
 
   // FIXME: To support emitting this build attribute as GCC does, the
diff --git a/llvm/lib/Target/ARM/ARMAsmPrinter.h b/llvm/lib/Target/ARM/ARMAsmPrinter.h
index 8a7ec4e2bcf2..2b067c753264 100644
--- a/llvm/lib/Target/ARM/ARMAsmPrinter.h
+++ b/llvm/lib/Target/ARM/ARMAsmPrinter.h
@@ -76,6 +76,8 @@ public:
     return "ARM Assembly Printer";
   }
 
+  const ARMBaseTargetMachine &getTM() const;
+
   void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
 
   void PrintSymbolOperand(const MachineOperand &MO, raw_ostream &O) override;
diff --git a/llvm/lib/Target/ARM/ARMFastISel.cpp b/llvm/lib/Target/ARM/ARMFastISel.cpp
index 765c65c5fcb2..06499a3945ee 100644
--- a/llvm/lib/Target/ARM/ARMFastISel.cpp
+++ b/llvm/lib/Target/ARM/ARMFastISel.cpp
@@ -20,6 +20,7 @@
 #include "ARMISelLowering.h"
 #include "ARMMachineFunctionInfo.h"
 #include "ARMSubtarget.h"
+#include "ARMTargetMachine.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMBaseInfo.h"
 #include "Utils/ARMBaseInfo.h"
@@ -134,9 +135,9 @@ class ARMFastISel final : public FastISel {
   /// make the right decision when generating code for different targets.
   const ARMSubtarget *Subtarget;
   Module &M;
-  const TargetMachine &TM;
-  const TargetInstrInfo &TII;
-  const TargetLowering &TLI;
+  const ARMBaseInstrInfo &TII;
+  const ARMTargetLowering &TLI;
+  const ARMBaseTargetMachine &TM;
   ARMFunctionInfo *AFI;
 
   // Convenience variables to avoid some queries.
@@ -149,8 +150,8 @@ class ARMFastISel final : public FastISel {
         : FastISel(funcInfo, libInfo),
           Subtarget(&funcInfo.MF->getSubtarget<ARMSubtarget>()),
           M(const_cast<Module &>(*funcInfo.Fn->getParent())),
-          TM(funcInfo.MF->getTarget()), TII(*Subtarget->getInstrInfo()),
-          TLI(*Subtarget->getTargetLowering()) {
+          TII(*Subtarget->getInstrInfo()), TLI(*Subtarget->getTargetLowering()),
+          TM(TLI.getTM()) {
       AFI = funcInfo.MF->getInfo<ARMFunctionInfo>();
       isThumb2 = AFI->isThumbFunction();
       Context = &funcInfo.Fn->getContext();
@@ -1893,7 +1894,7 @@ CCAssignFn *ARMFastISel::CCAssignFnForCall(CallingConv::ID CC,
     report_fatal_error("Unsupported calling convention");
   case CallingConv::Fast:
     if (Subtarget->hasVFP2Base() && !isVarArg) {
-      if (!Subtarget->isAAPCS_ABI())
+      if (!TM.isAAPCS_ABI())
         return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
       // For AAPCS ABI targets, just use VFP variant of the calling convention.
       return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
@@ -1902,7 +1903,7 @@ CCAssignFn *ARMFastISel::CCAssignFnForCall(CallingConv::ID CC,
   case CallingConv::C:
   case CallingConv::CXX_FAST_TLS:
     // Use target triple & subtarget features to do actual dispatch.
-    if (Subtarget->isAAPCS_ABI()) {
+    if (TM.isAAPCS_ABI()) {
       if (Subtarget->hasFPRegs() &&
           TM.Options.FloatABIType == FloatABI::Hard && !isVarArg)
         return (Return ? RetCC_ARM_AAPCS_VFP: CC_ARM_AAPCS_VFP);
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 05d8a1190ada..6e653687dbcb 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -499,9 +499,16 @@ void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
   setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
 }
 
-ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
+const ARMBaseTargetMachine &ARMTargetLowering::getTM() const {
+  return static_cast<const ARMBaseTargetMachine &>(getTargetMachine());
+}
+
+ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
                                      const ARMSubtarget &STI)
-    : TargetLowering(TM), Subtarget(&STI) {
+    : TargetLowering(TM_), Subtarget(&STI) {
+
+  const auto &TM = static_cast<const ARMBaseTargetMachine &>(TM_);
+
   RegInfo = Subtarget->getRegisterInfo();
   Itins = Subtarget->getInstrItineraryData();
 
@@ -591,7 +598,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   }
 
   // RTLIB
-  if (Subtarget->isAAPCS_ABI() &&
+  if (TM.isAAPCS_ABI() &&
       (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
        Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
     // clang-format off
@@ -716,7 +723,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
   // non-watchos platforms, but are needed for some targets which use a
   // hard-float calling convention by default.
   if (!Subtarget->isTargetWatchABI()) {
-    if (Subtarget->isAAPCS_ABI()) {
+    if (TM.isAAPCS_ABI()) {
       setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
       setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
       setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
@@ -2070,7 +2077,7 @@ ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
     return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
   case CallingConv::C:
   case CallingConv::Tail:
-    if (!Subtarget->isAAPCS_ABI())
+    if (!getTM().isAAPCS_ABI())
       return CallingConv::ARM_APCS;
     else if (Subtarget->hasFPRegs() && !Subtarget->isThumb1Only() &&
              getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
@@ -2080,12 +2087,12 @@ ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
       return CallingConv::ARM_AAPCS;
   case CallingConv::Fast:
   case CallingConv::CXX_FAST_TLS:
-    if (!Subtarget->isAAPCS_ABI()) {
+    if (!getTM().isAAPCS_ABI()) {
       if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && !isVarArg)
         return CallingConv::Fast;
       return CallingConv::ARM_APCS;
-    } else if (Subtarget->hasVFP2Base() &&
-               !Subtarget->isThumb1Only() && !isVarArg)
+    } else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() &&
+               !isVarArg)
       return CallingConv::ARM_AAPCS_VFP;
     else
       return CallingConv::ARM_AAPCS;
@@ -3273,7 +3280,7 @@ ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
     SDValue Arg = OutVals[realRVLocIdx];
     bool ReturnF16 = false;
 
-    if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
+    if (Subtarget->hasFullFP16() && getTM().isTargetHardFloat()) {
       // Half-precision return values can be returned like this:
       //
       // t11 f16 = fadd ...
@@ -9937,7 +9944,7 @@ SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
   auto &DL = DAG.getDataLayout();
 
   ArgListTy Args;
-  bool ShouldUseSRet = Subtarget->isAPCS_ABI();
+  bool ShouldUseSRet = getTM().isAPCS_ABI();
   SDValue SRet;
   if (ShouldUseSRet) {
     // Create stack object for sret.
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 357ca9ea5d20..9c330e60a7d5 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -34,6 +34,7 @@
 
 namespace llvm {
 
+class ARMBaseTargetMachine;
 class ARMSubtarget;
 class DataLayout;
 class FastISel;
@@ -414,6 +415,8 @@ class VectorType;
     explicit ARMTargetLowering(const TargetMachine &TM,
                                const ARMSubtarget &STI);
 
+    const ARMBaseTargetMachine &getTM() const;
+
     unsigned getJumpTableEncoding() const override;
     bool useSoftFloat() const override;
 
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp
index 91d385a0b595..abca4bb947bc 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.cpp
+++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp
@@ -201,9 +201,9 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   if (isTargetWindows())
     NoARM = true;
 
-  if (isAAPCS_ABI())
+  if (TM.isAAPCS_ABI())
     stackAlignment = Align(8);
-  if (isTargetNaCl() || isAAPCS16_ABI())
+  if (isTargetNaCl() || TM.isAAPCS16_ABI())
     stackAlignment = Align(16);
 
   // FIXME: Completely disable sibcall for Thumb1 since ThumbRegisterInfo::
@@ -320,22 +320,6 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
   }
 }
 
-bool ARMSubtarget::isTargetHardFloat() const { return TM.isTargetHardFloat(); }
-
-bool ARMSubtarget::isAPCS_ABI() const {
-  assert(TM.TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
-  return TM.TargetABI == ARMBaseTargetMachine::ARM_ABI_APCS;
-}
-bool ARMSubtarget::isAAPCS_ABI() const {
-  assert(TM.TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
-  return TM.TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS ||
-         TM.TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16;
-}
-bool ARMSubtarget::isAAPCS16_ABI() const {
-  assert(TM.TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
-  return TM.TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16;
-}
-
 bool ARMSubtarget::isROPI() const {
   return TM.getRelocationModel() == Reloc::ROPI ||
          TM.getRelocationModel() == Reloc::ROPI_RWPI;
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 890a22f574a6..7893796e313b 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -360,8 +360,6 @@ public:
     return TargetTriple.isTargetEHABICompatible();
   }
 
-  bool isTargetHardFloat() const;
-
   bool isReadTPSoft() const {
     return !(isReadTPTPIDRURW() || isReadTPTPIDRURO() || isReadTPTPIDRPRW());
   }
@@ -370,10 +368,6 @@ public:
 
   bool isXRaySupported() const override;
 
-  bool isAPCS_ABI() const;
-  bool isAAPCS_ABI() const;
-  bool isAAPCS16_ABI() const;
-
   bool isROPI() const;
   bool isRWPI() const;
 
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index fee77a44e5e8..0d947d924eb6 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -271,6 +271,22 @@ ARMBaseTargetMachine::ARMBaseTargetMachine(const Target &T, const Triple &TT,
 
 ARMBaseTargetMachine::~ARMBaseTargetMachine() = default;
 
+bool ARMBaseTargetMachine::isAPCS_ABI() const {
+  assert(TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
+  return TargetABI == ARMBaseTargetMachine::ARM_ABI_APCS;
+}
+
+bool ARMBaseTargetMachine::isAAPCS_ABI() const {
+  assert(TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
+  return TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS ||
+         TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16;
+}
+
+bool ARMBaseTargetMachine::isAAPCS16_ABI() const {
+  assert(TargetABI != ARMBaseTargetMachine::ARM_ABI_UNKNOWN);
+  return TargetABI == ARMBaseTargetMachine::ARM_ABI_AAPCS16;
+}
+
 MachineFunctionInfo *ARMBaseTargetMachine::createMachineFunctionInfo(
     BumpPtrAllocator &Allocator, const Function &F,
     const TargetSubtargetInfo *STI) const {
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h
index 99fd817c81f8..513fe713c0bc 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -66,6 +66,10 @@ public:
     return TLOF.get();
   }
 
+  bool isAPCS_ABI() const;
+  bool isAAPCS_ABI() const;
+  bool isAAPCS16_ABI() const;
+
   bool isTargetHardFloat() const {
     return TargetTriple.getEnvironment() == Triple::GNUEABIHF ||
            TargetTriple.getEnvironment() == Triple::GNUEABIHFT64 ||
-- 
cgit v1.2.3


From 6e5ee4aa98f1dc16e6a75a7fd298a59f1edd1c6e Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Wed, 18 Jun 2025 17:48:10 -0700
Subject: [RISCV] Save vector registers in interrupt handler. (#143808)

Corresponding gcc bug report
https://gcc.gnu.org/bugzilla/show_bug.cgi?id=110665

The generated code is pretty awful.
---
 llvm/lib/Target/RISCV/RISCVCallingConv.td          |   30 +-
 llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp        |   10 +
 llvm/test/CodeGen/RISCV/interrupt-attr.ll          | 6820 ++++++++++++++++++++
 .../CodeGen/RISCV/rvv/interrupt-attr-nocall.ll     |  502 ++
 4 files changed, 7360 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/RISCV/rvv/interrupt-attr-nocall.ll

diff --git a/llvm/lib/Target/RISCV/RISCVCallingConv.td b/llvm/lib/Target/RISCV/RISCVCallingConv.td
index 98e05b7f8eca..cbf039edec27 100644
--- a/llvm/lib/Target/RISCV/RISCVCallingConv.td
+++ b/llvm/lib/Target/RISCV/RISCVCallingConv.td
@@ -56,14 +56,40 @@ def CSR_XLEN_F32_Interrupt: CalleeSavedRegs<(add CSR_Interrupt,
 def CSR_XLEN_F64_Interrupt: CalleeSavedRegs<(add CSR_Interrupt,
                                              (sequence "F%u_D", 0, 31))>;
 
+// Same as CSR_Interrupt, but including all vector registers.
+def CSR_XLEN_V_Interrupt: CalleeSavedRegs<(add CSR_Interrupt,
+                                           (sequence "V%u", 0, 31))>;
+
+// Same as CSR_Interrupt, but including all 32-bit FP registers and all vector
+// registers.
+def CSR_XLEN_F32_V_Interrupt: CalleeSavedRegs<(add CSR_XLEN_F32_Interrupt,
+                                               (sequence "V%u", 0, 31))>;
+
+// Same as CSR_Interrupt, but including all 64-bit FP registers and all vector
+// registers.
+def CSR_XLEN_F64_V_Interrupt: CalleeSavedRegs<(add CSR_XLEN_F64_Interrupt,
+                                               (sequence "V%u", 0, 31))>;
+
 // Same as CSR_Interrupt, but excluding X16-X31.
 def CSR_Interrupt_RVE : CalleeSavedRegs<(sub CSR_Interrupt,
                                         (sequence "X%u", 16, 31))>;
 
 // Same as CSR_XLEN_F32_Interrupt, but excluding X16-X31.
 def CSR_XLEN_F32_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F32_Interrupt,
-                                                (sequence "X%u", 16, 31))>;
+                                                 (sequence "X%u", 16, 31))>;
 
 // Same as CSR_XLEN_F64_Interrupt, but excluding X16-X31.
 def CSR_XLEN_F64_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F64_Interrupt,
-                                                (sequence "X%u", 16, 31))>;
+                                                 (sequence "X%u", 16, 31))>;
+
+// Same as CSR_XLEN_V_Interrupt, but excluding X16-X31.
+def CSR_XLEN_V_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_V_Interrupt,
+                                               (sequence "X%u", 16, 31))>;
+
+// Same as CSR_XLEN_F32_V_Interrupt, but excluding X16-X31.
+def CSR_XLEN_F32_V_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F32_V_Interrupt,
+                                                   (sequence "X%u", 16, 31))>;
+
+// Same as CSR_XLEN_F64_V_Interrupt, but excluding X16-X31.
+def CSR_XLEN_F64_V_Interrupt_RVE: CalleeSavedRegs<(sub CSR_XLEN_F64_V_Interrupt,
+                                                   (sequence "X%u", 16, 31))>;
diff --git a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
index 112142e1ef2f..7fdbf4be1ed1 100644
--- a/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVRegisterInfo.cpp
@@ -69,6 +69,16 @@ RISCVRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   if (MF->getFunction().getCallingConv() == CallingConv::GHC)
     return CSR_NoRegs_SaveList;
   if (MF->getFunction().hasFnAttribute("interrupt")) {
+    if (Subtarget.hasVInstructions()) {
+      if (Subtarget.hasStdExtD())
+        return Subtarget.hasStdExtE() ? CSR_XLEN_F64_V_Interrupt_RVE_SaveList
+                                      : CSR_XLEN_F64_V_Interrupt_SaveList;
+      if (Subtarget.hasStdExtF())
+        return Subtarget.hasStdExtE() ? CSR_XLEN_F32_V_Interrupt_RVE_SaveList
+                                      : CSR_XLEN_F32_V_Interrupt_SaveList;
+      return Subtarget.hasStdExtE() ? CSR_XLEN_V_Interrupt_RVE_SaveList
+                                    : CSR_XLEN_V_Interrupt_SaveList;
+    }
     if (Subtarget.hasStdExtD())
       return Subtarget.hasStdExtE() ? CSR_XLEN_F64_Interrupt_RVE_SaveList
                                     : CSR_XLEN_F64_Interrupt_SaveList;
diff --git a/llvm/test/CodeGen/RISCV/interrupt-attr.ll b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
index ba20ba77e6b2..e278b8d0b53b 100644
--- a/llvm/test/CodeGen/RISCV/interrupt-attr.ll
+++ b/llvm/test/CodeGen/RISCV/interrupt-attr.ll
@@ -19,6 +19,13 @@
 ; RUN: 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-RV32E
 ; RUN: llc -mtriple riscv32-unknown-elf -mattr=+e,+f -o - %s \
 ; RUN: 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-RV32E-F
+
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+zve32x -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV32-V
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+zve32x,+f -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV32-FV
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+zve32x,+f,+d -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV32-FDV
 ;
 ; RUN: llc -mtriple riscv64-unknown-elf -o - %s \
 ; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV64
@@ -42,6 +49,13 @@
 ; RUN: 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-RV64E-F
 ; RUN: llc -mtriple riscv64-unknown-elf -mattr=+e,+f,+d -o - %s \
 ; RUN: 2>&1 | FileCheck %s -check-prefixes=CHECK,CHECK-RV64E-FD
+;
+; RUN: llc -mtriple riscv64-unknown-elf -mattr=+zve32x -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV64-V
+; RUN: llc -mtriple riscv64-unknown-elf -mattr=+zve32x,+f -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV64-FV
+; RUN: llc -mtriple riscv64-unknown-elf -mattr=+zve32x,+f,+d -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK -check-prefix CHECK-RV64-FDV
 
 ;
 ; Checking for special return instructions (sret, mret).
@@ -757,6 +771,1697 @@ define void @foo_with_call() #1 {
 ; CHECK-RV32E-F-NEXT:    addi sp, sp, 168
 ; CHECK-RV32E-F-NEXT:    mret
 ;
+; CHECK-RV32-V-LABEL: foo_with_call:
+; CHECK-RV32-V:       # %bb.0:
+; CHECK-RV32-V-NEXT:    addi sp, sp, -80
+; CHECK-RV32-V-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t0, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t1, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t2, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a1, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a2, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a3, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a4, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a5, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a6, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a7, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t3, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t4, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t5, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t6, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 5
+; CHECK-RV32-V-NEXT:    sub sp, sp, a0
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    addi a0, sp, 16
+; CHECK-RV32-V-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    call otherfoo
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    add a0, sp, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    addi a0, sp, 16
+; CHECK-RV32-V-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 5
+; CHECK-RV32-V-NEXT:    add sp, sp, a0
+; CHECK-RV32-V-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t0, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t1, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t2, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a1, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a2, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a3, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a4, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a5, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a6, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a7, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t3, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t4, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t5, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t6, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    addi sp, sp, 80
+; CHECK-RV32-V-NEXT:    mret
+;
+; CHECK-RV32-FV-LABEL: foo_with_call:
+; CHECK-RV32-FV:       # %bb.0:
+; CHECK-RV32-FV-NEXT:    addi sp, sp, -160
+; CHECK-RV32-FV-NEXT:    sw ra, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t0, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t1, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t2, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a0, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a1, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a2, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a3, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a4, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a5, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a6, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a7, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t3, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t4, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t5, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t6, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft0, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft1, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft2, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft3, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft4, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft5, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft6, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft7, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa1, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa2, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa3, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa4, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa5, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa6, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa7, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft8, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft9, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft10, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft11, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FV-NEXT:    sub sp, sp, a0
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    addi a0, sp, 16
+; CHECK-RV32-FV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    call otherfoo
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    addi a0, sp, 16
+; CHECK-RV32-FV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FV-NEXT:    add sp, sp, a0
+; CHECK-RV32-FV-NEXT:    lw ra, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t0, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t1, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t2, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a0, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a1, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a2, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a3, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a4, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a5, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a6, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a7, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t3, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t4, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t5, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t6, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft0, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft1, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft2, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft3, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft4, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft5, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft6, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft7, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa1, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa2, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa3, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa4, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa5, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa6, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa7, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft8, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft9, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft10, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft11, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    addi sp, sp, 160
+; CHECK-RV32-FV-NEXT:    mret
+;
+; CHECK-RV32-FDV-LABEL: foo_with_call:
+; CHECK-RV32-FDV:       # %bb.0:
+; CHECK-RV32-FDV-NEXT:    addi sp, sp, -240
+; CHECK-RV32-FDV-NEXT:    sw ra, 236(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t0, 232(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t1, 228(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t2, 224(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a0, 220(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a1, 216(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a2, 212(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a3, 208(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a4, 204(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a5, 200(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a6, 196(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a7, 192(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t3, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t4, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t5, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t6, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft0, 168(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft1, 160(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft2, 152(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft3, 144(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft4, 136(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft5, 128(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft6, 120(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft7, 112(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa0, 104(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa1, 96(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa2, 88(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa3, 80(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa4, 72(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa5, 64(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa6, 56(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa7, 48(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft8, 40(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft9, 32(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft10, 24(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft11, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub sp, sp, a0
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    addi a0, sp, 16
+; CHECK-RV32-FDV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    call otherfoo
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    addi a0, sp, 16
+; CHECK-RV32-FDV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FDV-NEXT:    add sp, sp, a0
+; CHECK-RV32-FDV-NEXT:    lw ra, 236(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t0, 232(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t1, 228(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t2, 224(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a0, 220(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a1, 216(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a2, 212(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a3, 208(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a4, 204(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a5, 200(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a6, 196(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a7, 192(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t3, 188(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t4, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t5, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t6, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft0, 168(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft1, 160(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft2, 152(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft3, 144(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft4, 136(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft5, 128(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft6, 120(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft7, 112(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa0, 104(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa1, 96(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa2, 88(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa3, 80(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa4, 72(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa5, 64(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa6, 56(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa7, 48(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft8, 40(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft9, 32(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft10, 24(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft11, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    addi sp, sp, 240
+; CHECK-RV32-FDV-NEXT:    mret
+;
 ; CHECK-RV64-LABEL: foo_with_call:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    addi sp, sp, -128
@@ -1530,6 +3235,1697 @@ define void @foo_with_call() #1 {
 ; CHECK-RV64E-FD-NEXT:    fld ft11, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64E-FD-NEXT:    addi sp, sp, 336
 ; CHECK-RV64E-FD-NEXT:    mret
+;
+; CHECK-RV64-V-LABEL: foo_with_call:
+; CHECK-RV64-V:       # %bb.0:
+; CHECK-RV64-V-NEXT:    addi sp, sp, -144
+; CHECK-RV64-V-NEXT:    sd ra, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t0, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t1, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t2, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a0, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a1, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a2, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a3, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a4, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a5, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a6, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a7, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t3, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t4, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t5, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t6, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 5
+; CHECK-RV64-V-NEXT:    sub sp, sp, a0
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    addi a0, sp, 16
+; CHECK-RV64-V-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    call otherfoo
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    add a0, sp, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    addi a0, sp, 16
+; CHECK-RV64-V-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 5
+; CHECK-RV64-V-NEXT:    add sp, sp, a0
+; CHECK-RV64-V-NEXT:    ld ra, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t0, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t1, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t2, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a0, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a1, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a2, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a3, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a4, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a5, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a6, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a7, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t3, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t4, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t5, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t6, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    addi sp, sp, 144
+; CHECK-RV64-V-NEXT:    mret
+;
+; CHECK-RV64-FV-LABEL: foo_with_call:
+; CHECK-RV64-FV:       # %bb.0:
+; CHECK-RV64-FV-NEXT:    addi sp, sp, -224
+; CHECK-RV64-FV-NEXT:    sd ra, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t0, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t1, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t2, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a0, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a1, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a2, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a3, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a4, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a5, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a6, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a7, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t3, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t4, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t5, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t6, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft0, 92(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft1, 88(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft2, 84(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft3, 80(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft4, 76(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft5, 72(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft6, 68(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft7, 64(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa1, 56(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa2, 52(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa3, 48(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa4, 44(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa5, 40(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa6, 36(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa7, 32(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft8, 28(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft9, 24(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft10, 20(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft11, 16(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FV-NEXT:    sub sp, sp, a0
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    addi a0, sp, 16
+; CHECK-RV64-FV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    call otherfoo
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    addi a0, sp, 16
+; CHECK-RV64-FV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FV-NEXT:    add sp, sp, a0
+; CHECK-RV64-FV-NEXT:    ld ra, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t0, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t1, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t2, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a0, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a1, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a2, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a3, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a4, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a5, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a6, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a7, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t3, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t4, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t5, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t6, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft0, 92(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft1, 88(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft2, 84(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft3, 80(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft4, 76(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft5, 72(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft6, 68(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft7, 64(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa1, 56(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa2, 52(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa3, 48(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa4, 44(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa5, 40(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa6, 36(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa7, 32(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft8, 28(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft9, 24(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft10, 20(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft11, 16(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    addi sp, sp, 224
+; CHECK-RV64-FV-NEXT:    mret
+;
+; CHECK-RV64-FDV-LABEL: foo_with_call:
+; CHECK-RV64-FDV:       # %bb.0:
+; CHECK-RV64-FDV-NEXT:    addi sp, sp, -304
+; CHECK-RV64-FDV-NEXT:    sd ra, 296(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t0, 288(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t1, 280(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t2, 272(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a0, 264(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a1, 256(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a2, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a3, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a4, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a5, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a6, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a7, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t3, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t4, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t5, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t6, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft0, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft1, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft2, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft3, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft4, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft5, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft6, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft7, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa0, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa1, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa2, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa3, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa4, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa5, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa6, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa7, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft8, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft9, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft10, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft11, 16(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub sp, sp, a0
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    addi a0, sp, 16
+; CHECK-RV64-FDV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    call otherfoo
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    add a0, sp, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    addi a0, sp, 16
+; CHECK-RV64-FDV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FDV-NEXT:    add sp, sp, a0
+; CHECK-RV64-FDV-NEXT:    ld ra, 296(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t0, 288(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t1, 280(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t2, 272(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a0, 264(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a1, 256(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a2, 248(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a3, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a4, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a5, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a6, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a7, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t3, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t4, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t5, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t6, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft0, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft1, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft2, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft3, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft4, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft5, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft6, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft7, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa0, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa1, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa2, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa3, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa4, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa5, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa6, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa7, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft8, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft9, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft10, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft11, 16(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    addi sp, sp, 304
+; CHECK-RV64-FDV-NEXT:    mret
   %call = call i32 @otherfoo()
   ret void
 }
@@ -2249,6 +5645,1718 @@ define void @foo_fp_with_call() #2 {
 ; CHECK-RV32E-F-NEXT:    addi sp, sp, 172
 ; CHECK-RV32E-F-NEXT:    mret
 ;
+; CHECK-RV32-V-LABEL: foo_fp_with_call:
+; CHECK-RV32-V:       # %bb.0:
+; CHECK-RV32-V-NEXT:    addi sp, sp, -80
+; CHECK-RV32-V-NEXT:    sw ra, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t0, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t1, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t2, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw s0, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw a7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t3, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t4, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t5, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    sw t6, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-V-NEXT:    addi s0, sp, 80
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 5
+; CHECK-RV32-V-NEXT:    sub sp, sp, a0
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-V-NEXT:    call otherfoo
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 4
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 4
+; CHECK-RV32-V-NEXT:    add a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 3
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 2
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    mv a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a1, a1, a0
+; CHECK-RV32-V-NEXT:    slli a0, a0, 1
+; CHECK-RV32-V-NEXT:    add a0, a0, a1
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a1, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, a1, a0
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    csrr a0, vlenb
+; CHECK-RV32-V-NEXT:    slli a0, a0, 5
+; CHECK-RV32-V-NEXT:    sub a0, s0, a0
+; CHECK-RV32-V-NEXT:    addi a0, a0, -80
+; CHECK-RV32-V-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-V-NEXT:    addi sp, s0, -80
+; CHECK-RV32-V-NEXT:    lw ra, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t0, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t1, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t2, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw s0, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw a7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t3, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t4, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t5, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    lw t6, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-V-NEXT:    addi sp, sp, 80
+; CHECK-RV32-V-NEXT:    mret
+;
+; CHECK-RV32-FV-LABEL: foo_fp_with_call:
+; CHECK-RV32-FV:       # %bb.0:
+; CHECK-RV32-FV-NEXT:    addi sp, sp, -160
+; CHECK-RV32-FV-NEXT:    sw ra, 156(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t0, 152(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t1, 148(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t2, 144(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw s0, 140(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a0, 136(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a1, 132(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a2, 128(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a3, 124(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a4, 120(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a5, 116(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a6, 112(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw a7, 108(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t3, 104(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t4, 100(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t5, 96(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    sw t6, 92(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft0, 88(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft1, 84(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft2, 80(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft3, 76(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft4, 72(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft5, 68(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft6, 64(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft7, 60(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa0, 56(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa1, 52(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa2, 48(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa3, 44(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa4, 40(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa5, 36(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa6, 32(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw fa7, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft8, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft9, 20(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft10, 16(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    fsw ft11, 12(sp) # 4-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    addi s0, sp, 160
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FV-NEXT:    sub sp, sp, a0
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FV-NEXT:    call otherfoo
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    mv a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FV-NEXT:    addi a0, a0, -160
+; CHECK-RV32-FV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    addi sp, s0, -160
+; CHECK-RV32-FV-NEXT:    lw ra, 156(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t0, 152(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t1, 148(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t2, 144(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw s0, 140(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a0, 136(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a1, 132(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a2, 128(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a3, 124(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a4, 120(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a5, 116(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a6, 112(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw a7, 108(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t3, 104(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t4, 100(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t5, 96(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    lw t6, 92(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft0, 88(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft1, 84(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft2, 80(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft3, 76(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft4, 72(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft5, 68(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft6, 64(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft7, 60(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa0, 56(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa1, 52(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa2, 48(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa3, 44(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa4, 40(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa5, 36(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa6, 32(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw fa7, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft8, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft9, 20(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft10, 16(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    flw ft11, 12(sp) # 4-byte Folded Reload
+; CHECK-RV32-FV-NEXT:    addi sp, sp, 160
+; CHECK-RV32-FV-NEXT:    mret
+;
+; CHECK-RV32-FDV-LABEL: foo_fp_with_call:
+; CHECK-RV32-FDV:       # %bb.0:
+; CHECK-RV32-FDV-NEXT:    addi sp, sp, -240
+; CHECK-RV32-FDV-NEXT:    sw ra, 236(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t0, 232(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t1, 228(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t2, 224(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw s0, 220(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a0, 216(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a1, 212(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a2, 208(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a3, 204(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a4, 200(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a5, 196(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a6, 192(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw a7, 188(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t3, 184(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t4, 180(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t5, 176(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    sw t6, 172(sp) # 4-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft0, 160(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft1, 152(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft2, 144(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft3, 136(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft4, 128(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft5, 120(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft6, 112(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft7, 104(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa0, 96(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa1, 88(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa2, 80(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa3, 72(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa4, 64(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa5, 56(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa6, 48(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd fa7, 40(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft8, 32(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft9, 24(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft10, 16(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    fsd ft11, 8(sp) # 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    addi s0, sp, 240
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub sp, sp, a0
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-FDV-NEXT:    call otherfoo
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV32-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    mv a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV32-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV32-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV32-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV32-FDV-NEXT:    addi a0, a0, -240
+; CHECK-RV32-FDV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    addi sp, s0, -240
+; CHECK-RV32-FDV-NEXT:    lw ra, 236(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t0, 232(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t1, 228(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t2, 224(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw s0, 220(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a0, 216(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a1, 212(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a2, 208(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a3, 204(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a4, 200(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a5, 196(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a6, 192(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw a7, 188(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t3, 184(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t4, 180(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t5, 176(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    lw t6, 172(sp) # 4-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft0, 160(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft1, 152(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft2, 144(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft3, 136(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft4, 128(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft5, 120(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft6, 112(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft7, 104(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa0, 96(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa1, 88(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa2, 80(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa3, 72(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa4, 64(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa5, 56(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa6, 48(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld fa7, 40(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft8, 32(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft9, 24(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft10, 16(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    fld ft11, 8(sp) # 8-byte Folded Reload
+; CHECK-RV32-FDV-NEXT:    addi sp, sp, 240
+; CHECK-RV32-FDV-NEXT:    mret
+;
 ; CHECK-RV64-LABEL: foo_fp_with_call:
 ; CHECK-RV64:       # %bb.0:
 ; CHECK-RV64-NEXT:    addi sp, sp, -144
@@ -3052,6 +8160,1718 @@ define void @foo_fp_with_call() #2 {
 ; CHECK-RV64E-FD-NEXT:    fld ft11, 0(sp) # 8-byte Folded Reload
 ; CHECK-RV64E-FD-NEXT:    addi sp, sp, 344
 ; CHECK-RV64E-FD-NEXT:    mret
+;
+; CHECK-RV64-V-LABEL: foo_fp_with_call:
+; CHECK-RV64-V:       # %bb.0:
+; CHECK-RV64-V-NEXT:    addi sp, sp, -160
+; CHECK-RV64-V-NEXT:    sd ra, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t0, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t1, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t2, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd s0, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a0, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a1, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a2, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a3, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a4, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a5, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a6, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd a7, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t3, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t4, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t5, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    sd t6, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    addi s0, sp, 160
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 5
+; CHECK-RV64-V-NEXT:    sub sp, sp, a0
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-V-NEXT:    call otherfoo
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 4
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 4
+; CHECK-RV64-V-NEXT:    add a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 3
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 2
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    mv a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a1, a1, a0
+; CHECK-RV64-V-NEXT:    slli a0, a0, 1
+; CHECK-RV64-V-NEXT:    add a0, a0, a1
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a1, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, a1, a0
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    csrr a0, vlenb
+; CHECK-RV64-V-NEXT:    slli a0, a0, 5
+; CHECK-RV64-V-NEXT:    sub a0, s0, a0
+; CHECK-RV64-V-NEXT:    addi a0, a0, -160
+; CHECK-RV64-V-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    addi sp, s0, -160
+; CHECK-RV64-V-NEXT:    ld ra, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t0, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t1, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t2, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld s0, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a0, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a1, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a2, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a3, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a4, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a5, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a6, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld a7, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t3, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t4, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t5, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    ld t6, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-V-NEXT:    addi sp, sp, 160
+; CHECK-RV64-V-NEXT:    mret
+;
+; CHECK-RV64-FV-LABEL: foo_fp_with_call:
+; CHECK-RV64-FV:       # %bb.0:
+; CHECK-RV64-FV-NEXT:    addi sp, sp, -240
+; CHECK-RV64-FV-NEXT:    sd ra, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t0, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t1, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t2, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd s0, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a0, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a1, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a2, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a3, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a4, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a5, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a6, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd a7, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t3, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t4, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t5, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    sd t6, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft0, 100(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft1, 96(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft2, 92(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft3, 88(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft4, 84(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft5, 80(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft6, 76(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft7, 72(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa0, 68(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa1, 64(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa2, 60(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa3, 56(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa4, 52(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa5, 48(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa6, 44(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw fa7, 40(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft8, 36(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft9, 32(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft10, 28(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    fsw ft11, 24(sp) # 4-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    addi s0, sp, 240
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FV-NEXT:    sub sp, sp, a0
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FV-NEXT:    call otherfoo
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    mv a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FV-NEXT:    addi a0, a0, -240
+; CHECK-RV64-FV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    addi sp, s0, -240
+; CHECK-RV64-FV-NEXT:    ld ra, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t0, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t1, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t2, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld s0, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a0, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a1, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a2, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a3, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a4, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a5, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a6, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld a7, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t3, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t4, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t5, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    ld t6, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft0, 100(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft1, 96(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft2, 92(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft3, 88(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft4, 84(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft5, 80(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft6, 76(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft7, 72(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa0, 68(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa1, 64(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa2, 60(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa3, 56(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa4, 52(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa5, 48(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa6, 44(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw fa7, 40(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft8, 36(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft9, 32(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft10, 28(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    flw ft11, 24(sp) # 4-byte Folded Reload
+; CHECK-RV64-FV-NEXT:    addi sp, sp, 240
+; CHECK-RV64-FV-NEXT:    mret
+;
+; CHECK-RV64-FDV-LABEL: foo_fp_with_call:
+; CHECK-RV64-FDV:       # %bb.0:
+; CHECK-RV64-FDV-NEXT:    addi sp, sp, -320
+; CHECK-RV64-FDV-NEXT:    sd ra, 312(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t0, 304(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t1, 296(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t2, 288(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd s0, 280(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a0, 272(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a1, 264(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a2, 256(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a3, 248(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a4, 240(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a5, 232(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a6, 224(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd a7, 216(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t3, 208(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t4, 200(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t5, 192(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    sd t6, 184(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft0, 176(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft1, 168(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft2, 160(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft3, 152(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft4, 144(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft5, 136(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft6, 128(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft7, 120(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa0, 112(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa1, 104(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa2, 96(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa3, 88(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa4, 80(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa5, 72(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa6, 64(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd fa7, 56(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft8, 48(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft9, 40(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft10, 32(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    fsd ft11, 24(sp) # 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    addi s0, sp, 320
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub sp, sp, a0
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v0, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v1, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v2, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v3, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v4, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v5, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v6, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v7, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v24, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v25, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v26, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v27, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v28, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v29, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v30, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vs1r.v v31, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV64-FDV-NEXT:    call otherfoo
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v0, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v1, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v2, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v3, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v4, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v5, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v6, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v7, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 4
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 4
+; CHECK-RV64-FDV-NEXT:    add a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 3
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v24, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v25, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v26, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v27, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 2
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v28, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    mv a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a1, a1, a0
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 1
+; CHECK-RV64-FDV-NEXT:    add a0, a0, a1
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v29, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a1, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, a1, a0
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v30, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    csrr a0, vlenb
+; CHECK-RV64-FDV-NEXT:    slli a0, a0, 5
+; CHECK-RV64-FDV-NEXT:    sub a0, s0, a0
+; CHECK-RV64-FDV-NEXT:    addi a0, a0, -320
+; CHECK-RV64-FDV-NEXT:    vl1r.v v31, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    addi sp, s0, -320
+; CHECK-RV64-FDV-NEXT:    ld ra, 312(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t0, 304(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t1, 296(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t2, 288(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld s0, 280(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a0, 272(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a1, 264(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a2, 256(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a3, 248(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a4, 240(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a5, 232(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a6, 224(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld a7, 216(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t3, 208(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t4, 200(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t5, 192(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    ld t6, 184(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft0, 176(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft1, 168(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft2, 160(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft3, 152(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft4, 144(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft5, 136(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft6, 128(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft7, 120(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa0, 112(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa1, 104(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa2, 96(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa3, 88(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa4, 80(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa5, 72(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa6, 64(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld fa7, 56(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft8, 48(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft9, 40(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft10, 32(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    fld ft11, 24(sp) # 8-byte Folded Reload
+; CHECK-RV64-FDV-NEXT:    addi sp, sp, 320
+; CHECK-RV64-FDV-NEXT:    mret
   %call = call i32 @otherfoo()
   ret void
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/interrupt-attr-nocall.ll b/llvm/test/CodeGen/RISCV/rvv/interrupt-attr-nocall.ll
new file mode 100644
index 000000000000..af2e8d384a44
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/interrupt-attr-nocall.ll
@@ -0,0 +1,502 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32-unknown-elf -mattr=+zve32x,+zvl128b -o - %s \
+; RUN: 2>&1 | FileCheck %s -check-prefix CHECK-RV32
+
+@a = external global <4 x i32>
+@b = external global <4 x i32>
+@c = external global <4 x i32>
+
+define void @foo_lmul1() nounwind #0 {
+; CHECK-RV32-LABEL: foo_lmul1:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -32
+; CHECK-RV32-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    sub sp, sp, a0
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    lui a0, %hi(a)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(a)
+; CHECK-RV32-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-RV32-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32-NEXT:    lui a0, %hi(b)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(b)
+; CHECK-RV32-NEXT:    vle32.v v9, (a0)
+; CHECK-RV32-NEXT:    vadd.vv v8, v9, v8
+; CHECK-RV32-NEXT:    lui a0, %hi(c)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(c)
+; CHECK-RV32-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 32
+; CHECK-RV32-NEXT:    mret
+  %1 = load <4 x i32>, ptr @a
+  %2 = load <4 x i32>, ptr @b
+  %add = add nsw <4 x i32> %2, %1
+  store <4 x i32> %add, ptr @c
+  ret void
+}
+
+@d = external global <8 x i32>
+@e = external global <8 x i32>
+@f = external global <8 x i32>
+
+define void @foo_lmul2() nounwind #0 {
+; CHECK-RV32-LABEL: foo_lmul2:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -32
+; CHECK-RV32-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    sub sp, sp, a0
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    lui a0, %hi(d)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(d)
+; CHECK-RV32-NEXT:    vsetivli zero, 8, e32, m2, ta, ma
+; CHECK-RV32-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32-NEXT:    lui a0, %hi(e)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(e)
+; CHECK-RV32-NEXT:    vle32.v v10, (a0)
+; CHECK-RV32-NEXT:    vadd.vv v8, v10, v8
+; CHECK-RV32-NEXT:    lui a0, %hi(f)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(f)
+; CHECK-RV32-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 32
+; CHECK-RV32-NEXT:    mret
+  %1 = load <8 x i32>, ptr @d
+  %2 = load <8 x i32>, ptr @e
+  %add = add nsw <8 x i32> %2, %1
+  store <8 x i32> %add, ptr @f
+  ret void
+}
+
+@g = external global <16 x i32>
+@h = external global <16 x i32>
+@i = external global <16 x i32>
+
+define void @foo_lmul4() nounwind #0 {
+; CHECK-RV32-LABEL: foo_lmul4:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -32
+; CHECK-RV32-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 3
+; CHECK-RV32-NEXT:    sub sp, sp, a0
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    slli a1, a0, 2
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    lui a0, %hi(g)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(g)
+; CHECK-RV32-NEXT:    vsetivli zero, 16, e32, m4, ta, ma
+; CHECK-RV32-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32-NEXT:    lui a0, %hi(h)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(h)
+; CHECK-RV32-NEXT:    vle32.v v12, (a0)
+; CHECK-RV32-NEXT:    vadd.vv v8, v12, v8
+; CHECK-RV32-NEXT:    lui a0, %hi(i)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(i)
+; CHECK-RV32-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32-NEXT:    sw a1, 4(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 2
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    lw a1, 4(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 3
+; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 32
+; CHECK-RV32-NEXT:    mret
+  %1 = load <16 x i32>, ptr @g
+  %2 = load <16 x i32>, ptr @h
+  %add = add nsw <16 x i32> %2, %1
+  store <16 x i32> %add, ptr @i
+  ret void
+}
+
+@j = external global <32 x i32>
+@k = external global <32 x i32>
+@l = external global <32 x i32>
+
+define void @foo_lmul8() nounwind #0 {
+; CHECK-RV32-LABEL: foo_lmul8:
+; CHECK-RV32:       # %bb.0:
+; CHECK-RV32-NEXT:    addi sp, sp, -32
+; CHECK-RV32-NEXT:    sw a0, 28(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    sw a1, 24(sp) # 4-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 4
+; CHECK-RV32-NEXT:    sub sp, sp, a0
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 4
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v8, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v9, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v10, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v11, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v12, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v13, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v14, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 3
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v15, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v16, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v17, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 2
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v18, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v19, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v20, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v21, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vs1r.v v22, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vs1r.v v23, (a0) # vscale x 8-byte Folded Spill
+; CHECK-RV32-NEXT:    lui a0, %hi(j)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(j)
+; CHECK-RV32-NEXT:    li a1, 32
+; CHECK-RV32-NEXT:    vsetvli zero, a1, e32, m8, ta, ma
+; CHECK-RV32-NEXT:    vle32.v v8, (a0)
+; CHECK-RV32-NEXT:    lui a0, %hi(k)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(k)
+; CHECK-RV32-NEXT:    vle32.v v16, (a0)
+; CHECK-RV32-NEXT:    vadd.vv v8, v16, v8
+; CHECK-RV32-NEXT:    lui a0, %hi(l)
+; CHECK-RV32-NEXT:    addi a0, a0, %lo(l)
+; CHECK-RV32-NEXT:    vse32.v v8, (a0)
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 4
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v8, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v9, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v10, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v11, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a1, a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v12, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v13, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v14, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 3
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v15, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 3
+; CHECK-RV32-NEXT:    sub a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v16, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    mv a1, a0
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, a0, a1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v17, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 2
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v18, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 2
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v19, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a1, a0, 1
+; CHECK-RV32-NEXT:    add a0, a1, a0
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v20, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 1
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v21, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    add a0, sp, a0
+; CHECK-RV32-NEXT:    addi a0, a0, 16
+; CHECK-RV32-NEXT:    vl1r.v v22, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    addi a0, sp, 16
+; CHECK-RV32-NEXT:    vl1r.v v23, (a0) # vscale x 8-byte Folded Reload
+; CHECK-RV32-NEXT:    csrr a0, vlenb
+; CHECK-RV32-NEXT:    slli a0, a0, 4
+; CHECK-RV32-NEXT:    add sp, sp, a0
+; CHECK-RV32-NEXT:    lw a0, 28(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    lw a1, 24(sp) # 4-byte Folded Reload
+; CHECK-RV32-NEXT:    addi sp, sp, 32
+; CHECK-RV32-NEXT:    mret
+  %1 = load <32 x i32>, ptr @j
+  %2 = load <32 x i32>, ptr @k
+  %add = add nsw <32 x i32> %2, %1
+  store <32 x i32> %add, ptr @l
+  ret void
+}
+
+attributes #0 = { "interrupt"="machine" }
-- 
cgit v1.2.3


From 2bcdfa198aa511479c46144c5cf95c7c685384ef Mon Sep 17 00:00:00 2001
From: Sirui Mu <msrlancern@gmail.com>
Date: Thu, 19 Jun 2025 09:58:19 +0800
Subject: [CIR] Add side effect attribute to call operations (#144201)

This patch adds `side_effect` attribute to `cir.call` operation.

Other function call attributes will be added in later patches.
---
 .../clang/CIR/Dialect/Builder/CIRBaseBuilder.h     | 16 ++++--
 clang/include/clang/CIR/Dialect/IR/CIRAttrs.td     |  5 ++
 clang/include/clang/CIR/Dialect/IR/CIROps.td       | 43 +++++++++++++-
 .../clang/CIR/Interfaces/CIROpInterfaces.td        |  2 +
 clang/include/clang/CIR/MissingFeatures.h          |  1 -
 clang/lib/CIR/CodeGen/CIRGenCall.cpp               | 46 ++++++++++++---
 clang/lib/CIR/CodeGen/CIRGenCall.h                 |  6 ++
 clang/lib/CIR/CodeGen/CIRGenModule.h               | 10 ++++
 clang/lib/CIR/Dialect/IR/CIRDialect.cpp            | 66 +++++++++++++++++++++-
 .../lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp  | 51 +++++++++++++++--
 clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h  |  5 ++
 clang/test/CIR/CodeGen/call.c                      | 26 +++++++++
 clang/test/CIR/IR/call.cir                         |  4 ++
 13 files changed, 257 insertions(+), 24 deletions(-)

diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
index 502d58d7db8b..3e052c564112 100644
--- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
+++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h
@@ -227,22 +227,26 @@ public:
   //===--------------------------------------------------------------------===//
 
   cir::CallOp createCallOp(mlir::Location loc, mlir::SymbolRefAttr callee,
-                           mlir::Type returnType, mlir::ValueRange operands) {
-    return create<cir::CallOp>(loc, callee, returnType, operands);
+                           mlir::Type returnType, mlir::ValueRange operands,
+                           cir::SideEffect sideEffect = cir::SideEffect::All) {
+    return create<cir::CallOp>(loc, callee, returnType, operands, sideEffect);
   }
 
   cir::CallOp createCallOp(mlir::Location loc, cir::FuncOp callee,
-                           mlir::ValueRange operands) {
+                           mlir::ValueRange operands,
+                           cir::SideEffect sideEffect = cir::SideEffect::All) {
     return createCallOp(loc, mlir::SymbolRefAttr::get(callee),
-                        callee.getFunctionType().getReturnType(), operands);
+                        callee.getFunctionType().getReturnType(), operands,
+                        sideEffect);
   }
 
   cir::CallOp createIndirectCallOp(mlir::Location loc,
                                    mlir::Value indirectTarget,
                                    cir::FuncType funcType,
-                                   mlir::ValueRange operands) {
+                                   mlir::ValueRange operands,
+                                   cir::SideEffect sideEffect) {
     return create<cir::CallOp>(loc, indirectTarget, funcType.getReturnType(),
-                               operands);
+                               operands, sideEffect);
   }
 
   //===--------------------------------------------------------------------===//
diff --git a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
index b48f4ed461cc..9e01dde379d7 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIRAttrs.td
@@ -42,6 +42,11 @@ class CIR_TypedAttr<string name, string attrMnemonic, list<Trait> traits = []>
   let assemblyFormat = [{}];
 }
 
+class CIR_I32EnumAttr<string name, string summary, list<I32EnumAttrCase> cases>
+    : I32EnumAttr<name, summary, cases> {
+  let cppNamespace = "::cir";
+}
+
 class CIRUnitAttr<string name, string attrMnemonic, list<Trait> traits = []>
     : CIR_Attr<name, attrMnemonic, traits> {
   let returnType = "bool";
diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td
index 4655cebc82ee..852d3aa13114 100644
--- a/clang/include/clang/CIR/Dialect/IR/CIROps.td
+++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td
@@ -1858,6 +1858,36 @@ def FuncOp : CIR_Op<"func", [
 // CallOp
 //===----------------------------------------------------------------------===//
 
+def CIR_SideEffect : CIR_I32EnumAttr<
+    "SideEffect", "allowed side effects of a function", [
+      I32EnumAttrCase<"All", 1, "all">,
+      I32EnumAttrCase<"Pure", 2, "pure">,
+      I32EnumAttrCase<"Const", 3, "const">
+    ]> {
+  let description = [{
+    The side effect attribute specifies the possible side effects of the callee
+    of a call operation. This is an enumeration attribute and all possible
+    enumerators are:
+
+    - all: The callee can have any side effects. This is the default if no side
+      effects are explicitly listed.
+    - pure: The callee may read data from memory, but it cannot write data to
+      memory. This has the same effect as the GNU C/C++ attribute
+      `__attribute__((pure))`.
+    - const: The callee may not read or write data from memory. This has the
+      same effect as the GNU C/C++ attribute `__attribute__((const))`.
+
+    Examples:
+
+    ```mlir
+    %2 = cir.call @add(%0, %1) : (!s32i, !s32i) -> !s32i
+    %2 = cir.call @add(%0, %1) : (!s32i, !s32i) -> !s32i side_effect(pure)
+    %2 = cir.call @add(%0, %1) : (!s32i, !s32i) -> !s32i side_effect(const)
+    ```
+  }];
+  let cppNamespace = "::cir";
+}
+
 class CIR_CallOpBase<string mnemonic, list<Trait> extra_traits = []>
     : Op<CIR_Dialect, mnemonic,
          !listconcat(extra_traits,
@@ -1911,7 +1941,8 @@ class CIR_CallOpBase<string mnemonic, list<Trait> extra_traits = []>
   // will add in the future.
 
   dag commonArgs = (ins OptionalAttr<FlatSymbolRefAttr>:$callee,
-      Variadic<CIR_AnyType>:$args);
+      Variadic<CIR_AnyType>:$args,
+      DefaultValuedAttr<CIR_SideEffect, "SideEffect::All">:$side_effect);
 }
 
 def CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
@@ -1942,20 +1973,26 @@ def CallOp : CIR_CallOpBase<"call", [NoRegionArguments]> {
   let builders = [
     // Build a call op for a direct call
     OpBuilder<(ins "mlir::SymbolRefAttr":$callee, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands), [{
+                   "mlir::ValueRange":$operands,
+                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
       assert(callee && "callee attribute is required for direct call");
       $_state.addOperands(operands);
       $_state.addAttribute("callee", callee);
+      $_state.addAttribute("side_effect",
+        SideEffectAttr::get($_builder.getContext(), sideEffect));
       if (resType && !isa<VoidType>(resType))
         $_state.addTypes(resType);
     }]>,
     // Build a call op for an indirect call
     OpBuilder<(ins "mlir::Value":$calleePtr, "mlir::Type":$resType,
-                   "mlir::ValueRange":$operands), [{
+                   "mlir::ValueRange":$operands,
+                   CArg<"SideEffect", "SideEffect::All">:$sideEffect), [{
       $_state.addOperands(calleePtr);
       $_state.addOperands(operands);
       if (resType && !isa<VoidType>(resType))
         $_state.addTypes(resType);
+      $_state.addAttribute("side_effect",
+        SideEffectAttr::get($_builder.getContext(), sideEffect));
     }]>,
   ];
 }
diff --git a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
index 80d78b11c2ba..203e42f7c575 100644
--- a/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
+++ b/clang/include/clang/CIR/Interfaces/CIROpInterfaces.td
@@ -34,6 +34,8 @@ let cppNamespace = "::cir" in {
           "Return the number of operands, accounts for indirect call or "
           "exception info",
           "unsigned", "getNumArgOperands", (ins)>,
+      InterfaceMethod<"Return the side effects of the call operation",
+                      "cir::SideEffect", "getSideEffect", (ins)>,
     ];
   }
 
diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 3d120903dea1..45452c5929a3 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -95,7 +95,6 @@ struct MissingFeatures {
   static bool opCallReturn() { return false; }
   static bool opCallArgEvaluationOrder() { return false; }
   static bool opCallCallConv() { return false; }
-  static bool opCallSideEffect() { return false; }
   static bool opCallNoPrototypeFunc() { return false; }
   static bool opCallMustTail() { return false; }
   static bool opCallVirtual() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.cpp b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
index af0e6ca822b8..9c9c96604c16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.cpp
@@ -77,6 +77,35 @@ void CIRGenFunction::emitAggregateStore(mlir::Value value, Address dest) {
   builder.createStore(*currSrcLoc, value, dest);
 }
 
+/// Construct the CIR attribute list of a function or call.
+void CIRGenModule::constructAttributeList(CIRGenCalleeInfo calleeInfo,
+                                          cir::SideEffect &sideEffect) {
+  assert(!cir::MissingFeatures::opCallCallConv());
+  sideEffect = cir::SideEffect::All;
+
+  assert(!cir::MissingFeatures::opCallAttrs());
+
+  const Decl *targetDecl = calleeInfo.getCalleeDecl().getDecl();
+
+  if (targetDecl) {
+    assert(!cir::MissingFeatures::opCallAttrs());
+
+    // 'const', 'pure' and 'noalias' attributed functions are also nounwind.
+    if (targetDecl->hasAttr<ConstAttr>()) {
+      // gcc specifies that 'const' functions have greater restrictions than
+      // 'pure' functions, so they also cannot have infinite loops.
+      sideEffect = cir::SideEffect::Const;
+    } else if (targetDecl->hasAttr<PureAttr>()) {
+      // gcc specifies that 'pure' functions cannot have infinite loops.
+      sideEffect = cir::SideEffect::Pure;
+    }
+
+    assert(!cir::MissingFeatures::opCallAttrs());
+  }
+
+  assert(!cir::MissingFeatures::opCallAttrs());
+}
+
 /// Returns the canonical formal type of the given C++ method.
 static CanQual<FunctionProtoType> getFormalType(const CXXMethodDecl *md) {
   return md->getType()
@@ -386,7 +415,8 @@ static cir::CIRCallOpInterface
 emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
                cir::FuncType indirectFuncTy, mlir::Value indirectFuncVal,
                cir::FuncOp directFuncOp,
-               const SmallVectorImpl<mlir::Value> &cirCallArgs) {
+               const SmallVectorImpl<mlir::Value> &cirCallArgs,
+               cir::SideEffect sideEffect) {
   CIRGenBuilderTy &builder = cgf.getBuilder();
 
   assert(!cir::MissingFeatures::opCallSurroundingTry());
@@ -397,11 +427,11 @@ emitCallLikeOp(CIRGenFunction &cgf, mlir::Location callLoc,
   if (indirectFuncTy) {
     // TODO(cir): Set calling convention for indirect calls.
     assert(!cir::MissingFeatures::opCallCallConv());
-    return builder.createIndirectCallOp(callLoc, indirectFuncVal,
-                                        indirectFuncTy, cirCallArgs);
+    return builder.createIndirectCallOp(
+        callLoc, indirectFuncVal, indirectFuncTy, cirCallArgs, sideEffect);
   }
 
-  return builder.createCallOp(callLoc, directFuncOp, cirCallArgs);
+  return builder.createCallOp(callLoc, directFuncOp, cirCallArgs, sideEffect);
 }
 
 const CIRGenFunctionInfo &
@@ -513,8 +543,9 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
     funcName = calleeFuncOp.getName();
 
   assert(!cir::MissingFeatures::opCallCallConv());
-  assert(!cir::MissingFeatures::opCallSideEffect());
   assert(!cir::MissingFeatures::opCallAttrs());
+  cir::SideEffect sideEffect;
+  cgm.constructAttributeList(callee.getAbstractInfo(), sideEffect);
 
   assert(!cir::MissingFeatures::invokeOp());
 
@@ -538,8 +569,9 @@ RValue CIRGenFunction::emitCall(const CIRGenFunctionInfo &funcInfo,
   assert(!cir::MissingFeatures::opCallAttrs());
 
   mlir::Location callLoc = loc;
-  cir::CIRCallOpInterface theCall = emitCallLikeOp(
-      *this, loc, indirectFuncTy, indirectFuncVal, directFuncOp, cirCallArgs);
+  cir::CIRCallOpInterface theCall =
+      emitCallLikeOp(*this, loc, indirectFuncTy, indirectFuncVal, directFuncOp,
+                     cirCallArgs, sideEffect);
 
   if (callOp)
     *callOp = theCall;
diff --git a/clang/lib/CIR/CodeGen/CIRGenCall.h b/clang/lib/CIR/CodeGen/CIRGenCall.h
index 0353848f3ec0..56c76c51a46d 100644
--- a/clang/lib/CIR/CodeGen/CIRGenCall.h
+++ b/clang/lib/CIR/CodeGen/CIRGenCall.h
@@ -105,6 +105,12 @@ public:
   /// callee
   CIRGenCallee prepareConcreteCallee(CIRGenFunction &cgf) const;
 
+  CIRGenCalleeInfo getAbstractInfo() const {
+    assert(!cir::MissingFeatures::opCallVirtual());
+    assert(isOrdinary());
+    return abstractInfo;
+  }
+
   mlir::Operation *getFunctionPointer() const {
     assert(isOrdinary());
     return reinterpret_cast<mlir::Operation *>(kindOrFunctionPtr);
diff --git a/clang/lib/CIR/CodeGen/CIRGenModule.h b/clang/lib/CIR/CodeGen/CIRGenModule.h
index 0ea2d9f9c822..71806e3c5de2 100644
--- a/clang/lib/CIR/CodeGen/CIRGenModule.h
+++ b/clang/lib/CIR/CodeGen/CIRGenModule.h
@@ -14,6 +14,7 @@
 #define LLVM_CLANG_LIB_CIR_CODEGEN_CIRGENMODULE_H
 
 #include "CIRGenBuilder.h"
+#include "CIRGenCall.h"
 #include "CIRGenTypeCache.h"
 #include "CIRGenTypes.h"
 #include "CIRGenValue.h"
@@ -158,6 +159,15 @@ public:
       const CXXRecordDecl *derivedClass,
       llvm::iterator_range<CastExpr::path_const_iterator> path);
 
+  /// Get the CIR attributes and calling convention to use for a particular
+  /// function type.
+  ///
+  /// \param calleeInfo - The callee information these attributes are being
+  /// constructed for. If valid, the attributes applied to this decl may
+  /// contribute to the function attributes and calling convention.
+  void constructAttributeList(CIRGenCalleeInfo calleeInfo,
+                              cir::SideEffect &sideEffect);
+
   /// Return a constant array for the given string.
   mlir::Attribute getConstantArrayFromStringLiteral(const StringLiteral *e);
 
diff --git a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
index 3fcb0213b219..16248059c497 100644
--- a/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
+++ b/clang/lib/CIR/Dialect/IR/CIRDialect.cpp
@@ -92,6 +92,46 @@ Operation *cir::CIRDialect::materializeConstant(mlir::OpBuilder &builder,
 // Helpers
 //===----------------------------------------------------------------------===//
 
+// Parses one of the keywords provided in the list `keywords` and returns the
+// position of the parsed keyword in the list. If none of the keywords from the
+// list is parsed, returns -1.
+static int parseOptionalKeywordAlternative(AsmParser &parser,
+                                           ArrayRef<llvm::StringRef> keywords) {
+  for (auto en : llvm::enumerate(keywords)) {
+    if (succeeded(parser.parseOptionalKeyword(en.value())))
+      return en.index();
+  }
+  return -1;
+}
+
+namespace {
+template <typename Ty> struct EnumTraits {};
+
+#define REGISTER_ENUM_TYPE(Ty)                                                 \
+  template <> struct EnumTraits<cir::Ty> {                                     \
+    static llvm::StringRef stringify(cir::Ty value) {                          \
+      return stringify##Ty(value);                                             \
+    }                                                                          \
+    static unsigned getMaxEnumVal() { return cir::getMaxEnumValFor##Ty(); }    \
+  }
+
+REGISTER_ENUM_TYPE(SideEffect);
+} // namespace
+
+/// Parse an enum from the keyword, return failure if the keyword is not found.
+template <typename EnumTy, typename RetTy = EnumTy>
+static ParseResult parseCIRKeyword(AsmParser &parser, RetTy &result) {
+  llvm::SmallVector<llvm::StringRef, 10> names;
+  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
+    names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
+
+  int index = parseOptionalKeywordAlternative(parser, names);
+  if (index == -1)
+    return failure();
+  result = static_cast<RetTy>(index);
+  return success();
+}
+
 // Check if a region's termination omission is valid and, if so, creates and
 // inserts the omitted terminator into the region.
 static LogicalResult ensureRegionTerm(OpAsmParser &parser, Region &region,
@@ -534,6 +574,18 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
   if (parser.parseRParen())
     return mlir::failure();
 
+  if (parser.parseOptionalKeyword("side_effect").succeeded()) {
+    if (parser.parseLParen().failed())
+      return failure();
+    cir::SideEffect sideEffect;
+    if (parseCIRKeyword<cir::SideEffect>(parser, sideEffect).failed())
+      return failure();
+    if (parser.parseRParen().failed())
+      return failure();
+    auto attr = cir::SideEffectAttr::get(parser.getContext(), sideEffect);
+    result.addAttribute("side_effect", attr);
+  }
+
   if (parser.parseOptionalAttrDict(result.attributes))
     return ::mlir::failure();
 
@@ -556,7 +608,8 @@ static mlir::ParseResult parseCallCommon(mlir::OpAsmParser &parser,
 static void printCallCommon(mlir::Operation *op,
                             mlir::FlatSymbolRefAttr calleeSym,
                             mlir::Value indirectCallee,
-                            mlir::OpAsmPrinter &printer) {
+                            mlir::OpAsmPrinter &printer,
+                            cir::SideEffect sideEffect) {
   printer << ' ';
 
   auto callLikeOp = mlir::cast<cir::CIRCallOpInterface>(op);
@@ -572,7 +625,13 @@ static void printCallCommon(mlir::Operation *op,
   }
   printer << "(" << ops << ")";
 
-  printer.printOptionalAttrDict(op->getAttrs(), {"callee"});
+  if (sideEffect != cir::SideEffect::All) {
+    printer << " side_effect(";
+    printer << stringifySideEffect(sideEffect);
+    printer << ")";
+  }
+
+  printer.printOptionalAttrDict(op->getAttrs(), {"callee", "side_effect"});
 
   printer << " : ";
   printer.printFunctionalType(op->getOperands().getTypes(),
@@ -586,7 +645,8 @@ mlir::ParseResult cir::CallOp::parse(mlir::OpAsmParser &parser,
 
 void cir::CallOp::print(mlir::OpAsmPrinter &p) {
   mlir::Value indirectCallee = isIndirect() ? getIndirectCall() : nullptr;
-  printCallCommon(*this, getCalleeAttr(), indirectCallee, p);
+  cir::SideEffect sideEffect = getSideEffect();
+  printCallCommon(*this, getCalleeAttr(), indirectCallee, p, sideEffect);
 }
 
 static LogicalResult
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index a96501ab2c38..b73cb839828e 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -220,6 +220,39 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
   return value;
 }
 
+void convertSideEffectForCall(mlir::Operation *callOp,
+                              cir::SideEffect sideEffect,
+                              mlir::LLVM::MemoryEffectsAttr &memoryEffect,
+                              bool &noUnwind, bool &willReturn) {
+  using mlir::LLVM::ModRefInfo;
+
+  switch (sideEffect) {
+  case cir::SideEffect::All:
+    memoryEffect = {};
+    noUnwind = false;
+    willReturn = false;
+    break;
+
+  case cir::SideEffect::Pure:
+    memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
+        callOp->getContext(), /*other=*/ModRefInfo::Ref,
+        /*argMem=*/ModRefInfo::Ref,
+        /*inaccessibleMem=*/ModRefInfo::Ref);
+    noUnwind = true;
+    willReturn = true;
+    break;
+
+  case cir::SideEffect::Const:
+    memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
+        callOp->getContext(), /*other=*/ModRefInfo::NoModRef,
+        /*argMem=*/ModRefInfo::NoModRef,
+        /*inaccessibleMem=*/ModRefInfo::NoModRef);
+    noUnwind = true;
+    willReturn = true;
+    break;
+  }
+}
+
 /// IntAttr visitor.
 mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
   mlir::Location loc = parentOp->getLoc();
@@ -745,12 +778,18 @@ rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
                     mlir::FlatSymbolRefAttr calleeAttr) {
   llvm::SmallVector<mlir::Type, 8> llvmResults;
   mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
+  auto call = cast<cir::CIRCallOpInterface>(op);
 
   if (converter->convertTypes(cirResults, llvmResults).failed())
     return mlir::failure();
 
   assert(!cir::MissingFeatures::opCallCallConv());
-  assert(!cir::MissingFeatures::opCallSideEffect());
+
+  mlir::LLVM::MemoryEffectsAttr memoryEffects;
+  bool noUnwind = false;
+  bool willReturn = false;
+  convertSideEffectForCall(op, call.getSideEffect(), memoryEffects, noUnwind,
+                           willReturn);
 
   mlir::LLVM::LLVMFunctionType llvmFnTy;
   if (calleeAttr) { // direct call
@@ -775,10 +814,14 @@ rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
   assert(!cir::MissingFeatures::opCallLandingPad());
   assert(!cir::MissingFeatures::opCallContinueBlock());
   assert(!cir::MissingFeatures::opCallCallConv());
-  assert(!cir::MissingFeatures::opCallSideEffect());
 
-  rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(op, llvmFnTy, calleeAttr,
-                                                  callOperands);
+  auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
+      op, llvmFnTy, calleeAttr, callOperands);
+  if (memoryEffects)
+    newOp.setMemoryEffectsAttr(memoryEffects);
+  newOp.setNoUnwind(noUnwind);
+  newOp.setWillReturn(willReturn);
+
   return mlir::success();
 }
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
index a80c66ac1abf..ae7247332c66 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.h
@@ -29,6 +29,11 @@ mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp, mlir::Attribute attr,
 
 mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage);
 
+void convertSideEffectForCall(mlir::Operation *callOp,
+                              cir::SideEffect sideEffect,
+                              mlir::LLVM::MemoryEffectsAttr &memoryEffect,
+                              bool &noUnwind, bool &willReturn);
+
 class CIRToLLVMAssumeOpLowering
     : public mlir::OpConversionPattern<cir::AssumeOp> {
 public:
diff --git a/clang/test/CIR/CodeGen/call.c b/clang/test/CIR/CodeGen/call.c
index 13f3c5a21ceb..f6aa41df7439 100644
--- a/clang/test/CIR/CodeGen/call.c
+++ b/clang/test/CIR/CodeGen/call.c
@@ -109,3 +109,29 @@ void f9() {
 // OGCG-NEXT:    store i64 %[[RET]], ptr %[[SLOT]], align 4
 // OGCG-NEXT:    %[[ARG:.+]] = load i64, ptr %[[SLOT]], align 4
 // OGCG-NEXT:    call void @f1(i64 %[[ARG]])
+
+__attribute__((pure)) int f10(int);
+__attribute__((const)) int f11(int);
+int f12(void) {
+  return f10(1) + f11(2);
+}
+
+// CIR-LABEL: cir.func @f12() -> !s32i
+// CIR:         %[[A:.+]] = cir.const #cir.int<1> : !s32i
+// CIR-NEXT:    %{{.+}} = cir.call @f10(%[[A]]) side_effect(pure) : (!s32i) -> !s32i
+// CIR-NEXT:    %[[B:.+]] = cir.const #cir.int<2> : !s32i
+// CIR-NEXT:    %{{.+}} = cir.call @f11(%[[B]]) side_effect(const) : (!s32i) -> !s32i
+
+// LLVM-LABEL: define i32 @f12()
+// LLVM:         %{{.+}} = call i32 @f10(i32 1) #[[ATTR0:.+]]
+// LLVM-NEXT:    %{{.+}} = call i32 @f11(i32 2) #[[ATTR1:.+]]
+
+// OGCG-LABEL: define dso_local i32 @f12()
+// OGCG:         %{{.+}} = call i32 @f10(i32 noundef 1) #[[ATTR0:.+]]
+// OGCG-NEXT:    %{{.+}} = call i32 @f11(i32 noundef 2) #[[ATTR1:.+]]
+
+// LLVM: attributes #[[ATTR0]] = { nounwind willreturn memory(read, errnomem: none) }
+// LLVM: attributes #[[ATTR1]] = { nounwind willreturn memory(none) }
+
+// OGCG: attributes #[[ATTR0]] = { nounwind willreturn memory(read) }
+// OGCG: attributes #[[ATTR1]] = { nounwind willreturn memory(none) }
diff --git a/clang/test/CIR/IR/call.cir b/clang/test/CIR/IR/call.cir
index e35c201b6ed4..5f0916775479 100644
--- a/clang/test/CIR/IR/call.cir
+++ b/clang/test/CIR/IR/call.cir
@@ -8,11 +8,15 @@ cir.func @f1()
 
 cir.func @f2() {
   cir.call @f1() : () -> ()
+  cir.call @f1() side_effect(pure) : () -> ()
+  cir.call @f1() side_effect(const) : () -> ()
   cir.return
 }
 
 // CHECK:      cir.func @f2() {
 // CHECK-NEXT:   cir.call @f1() : () -> ()
+// CHECK-NEXT:   cir.call @f1() side_effect(pure) : () -> ()
+// CHECK-NEXT:   cir.call @f1() side_effect(const) : () -> ()
 // CHECK-NEXT:   cir.return
 // CHECK-NEXT: }
 
-- 
cgit v1.2.3


From faf9295f4e3a23a972d29e2be85052beef409d08 Mon Sep 17 00:00:00 2001
From: MingYan <99472920+NexMing@users.noreply.github.com>
Date: Thu, 19 Jun 2025 10:15:31 +0800
Subject: [RISCV] Fix a bug where AVL is the last MI in MBB. (#144668)

When `AVL` is the last MI, `std::next(II)` equals `MBB.end()`, and
calling `II->getParent()` at that point will cause an error.

---------

Co-authored-by: yanming <ming.yan@terapines.com>
---
 llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp       | 15 +++++-----
 .../CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir   | 32 ++++++++++++++++++++++
 2 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 9a513891b765..78d64ea67324 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1119,25 +1119,26 @@ void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
     LIS->InsertMachineInstrInMaps(*MI);
     LiveInterval &LI = LIS->getInterval(AVLReg);
     SlotIndex SI = LIS->getInstructionIndex(*MI).getRegSlot();
+    const VNInfo *CurVNI = Info.getAVLVNInfo();
     // If the AVL value isn't live at MI, do a quick check to see if it's easily
     // extendable. Otherwise, we need to copy it.
-    if (LI.getVNInfoBefore(SI) != Info.getAVLVNInfo()) {
+    if (LI.getVNInfoBefore(SI) != CurVNI) {
       if (!LI.liveAt(SI) && LI.containsOneValue())
         LIS->extendToIndices(LI, SI);
       else {
         Register AVLCopyReg =
             MRI->createVirtualRegister(&RISCV::GPRNoX0RegClass);
+        MachineBasicBlock *MBB = LIS->getMBBFromIndex(CurVNI->def);
         MachineBasicBlock::iterator II;
-        if (Info.getAVLVNInfo()->isPHIDef())
-          II = LIS->getMBBFromIndex(Info.getAVLVNInfo()->def)->getFirstNonPHI();
+        if (CurVNI->isPHIDef())
+          II = MBB->getFirstNonPHI();
         else {
-          II = LIS->getInstructionFromIndex(Info.getAVLVNInfo()->def);
+          II = LIS->getInstructionFromIndex(CurVNI->def);
           II = std::next(II);
         }
         assert(II.isValid());
-        auto AVLCopy =
-            BuildMI(*II->getParent(), II, DL, TII->get(RISCV::COPY), AVLCopyReg)
-                .addReg(AVLReg);
+        auto AVLCopy = BuildMI(*MBB, II, DL, TII->get(RISCV::COPY), AVLCopyReg)
+                           .addReg(AVLReg);
         LIS->InsertMachineInstrInMaps(*AVLCopy);
         MI->getOperand(1).setReg(AVLCopyReg);
         LIS->createAndComputeVirtRegInterval(AVLCopyReg);
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
index 140875c4b24a..e09fc1828fec 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert-crossbb.mir
@@ -142,6 +142,10 @@
     ret void
   }
 
+  define void @avl_is_last_instr() {
+    ret void
+  }
+
   declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
 
   declare <vscale x 1 x i64> @llvm.riscv.vadd.nxv1i64.nxv1i64.i64(<vscale x 1 x i64>, <vscale x 1 x i64>, <vscale x 1 x i64>, i64) #1
@@ -1099,3 +1103,31 @@ body: |
     renamable $v10m2 = PseudoVADD_VV_M2 undef renamable $v10m2, %v, %v, -1, 5, 0
     renamable $v8m2 = PseudoVADD_VV_M2 undef renamable $v8m2, killed renamable $v10m2, killed %v, %outvl:gprnox0, 5, 0
     PseudoRET implicit $v8m2
+...
+---
+name: avl_is_last_instr
+tracksRegLiveness: true
+body: |
+  ; CHECK-LABEL: name: avl_is_last_instr
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x10
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   %avl:gprnox0 = COPY $x10
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gprnox0 = COPY %avl
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   dead %avl:gprnox0 = ADDI %avl, -1
+  ; CHECK-NEXT:   dead $x0 = PseudoVSETIVLI 1, 192 /* e8, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+  ; CHECK-NEXT:   $v8 = PseudoVMV_S_X undef renamable $v8, $x0, 1, 3 /* e8 */, implicit $vl, implicit $vtype
+  ; CHECK-NEXT:   dead $x0 = PseudoVSETVLI [[COPY]], 192 /* e8, m1, ta, ma */, implicit-def $vl, implicit-def $vtype
+  ; CHECK-NEXT:   $v8 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, $noreg, 3 /* e8 */, 3 /* ta, ma */, implicit $vl, implicit $vtype
+  bb.0:
+    liveins: $x10
+    %avl:gprnox0 = COPY $x10
+
+  bb.1:
+    %vl:gprnox0 = PseudoVSETVLI %avl:gprnox0, 192, implicit-def dead $vl, implicit-def dead $vtype
+    %avl:gprnox0 = ADDI %avl:gprnox0, -1
+    $v8 = PseudoVMV_S_X undef renamable $v8, $x0, 1, 3
+    $v8 = PseudoVADD_VV_M1 $noreg, $noreg, $noreg, %vl:gprnox0, 3, 3
-- 
cgit v1.2.3


From bfee625821c07d9a05b48e4a8b0f3d73c1233107 Mon Sep 17 00:00:00 2001
From: Durgadoss R <durgadossr@nvidia.com>
Date: Thu, 19 Jun 2025 07:49:08 +0530
Subject: [NVPTX] Attach Range attr to setmaxnreg and fence intrinsics
 (#144120)

This patch attaches the range attribute to the setmaxnreg
and fence.proxy.tensormap.* intrinsics. The range checks
are now handled generically in the Verifier. So, this patch
removes the per-intrinsic error-handling for range-checks
from the Verifier.

This patch also adds more coverage tests for these cases.

Signed-off-by: Durgadoss R <durgadossr@nvidia.com>
---
 llvm/include/llvm/IR/IntrinsicsNVVM.td            |  8 ++++++--
 llvm/lib/IR/Verifier.cpp                          | 10 ----------
 llvm/test/Verifier/NVPTX/fence-proxy.tensormap.ll | 17 +++++++++++++++++
 llvm/test/Verifier/NVPTX/setmaxnreg.ll            |  4 +++-
 4 files changed, 26 insertions(+), 13 deletions(-)
 create mode 100644 llvm/test/Verifier/NVPTX/fence-proxy.tensormap.ll

diff --git a/llvm/include/llvm/IR/IntrinsicsNVVM.td b/llvm/include/llvm/IR/IntrinsicsNVVM.td
index 4efdff71c016..410a0dea2bf5 100644
--- a/llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ b/llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -1341,9 +1341,11 @@ foreach scope = ["cta", "cluster", "gpu", "sys"] in {
         Intrinsic<[], [], [IntrNoCallback],
         "llvm.nvvm.fence.proxy.tensormap_generic.release." # scope>;
 
+  // The imm-arg 'size' can only be 128.
   def int_nvvm_fence_proxy_tensormap_generic_acquire_ # scope :
         Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
-                  [IntrNoCallback, IntrArgMemOnly, ImmArg<ArgIndex<1>>],
+                  [IntrNoCallback, IntrArgMemOnly, ImmArg<ArgIndex<1>>,
+                   Range<ArgIndex<1>, 128, 129>],
                   "llvm.nvvm.fence.proxy.tensormap_generic.acquire." # scope>;
 }
 
@@ -1989,10 +1991,12 @@ def int_nvvm_is_explicit_cluster
               "llvm.nvvm.is_explicit_cluster">;
 
 // Setmaxnreg inc/dec intrinsics
+// The imm-arg should be in the range: 24 <= val <= 256
 foreach op = ["dec", "inc"] in
   def int_nvvm_setmaxnreg_ # op # _sync_aligned_u32
     : DefaultAttrsIntrinsic<[], [llvm_i32_ty],
-              [IntrConvergent, IntrNoMem, IntrHasSideEffects, ImmArg<ArgIndex<0>>]>;
+              [IntrConvergent, IntrNoMem, IntrHasSideEffects,
+               ImmArg<ArgIndex<0>>, Range<ArgIndex<0>, 24, 257>]>;
 
 // Exit
 def int_nvvm_exit : NVVMBuiltin,
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 1f1041b25973..f0a4d7b6a4c1 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6557,8 +6557,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
     unsigned RegCount = cast<ConstantInt>(V)->getZExtValue();
     Check(RegCount % 8 == 0,
           "reg_count argument to nvvm.setmaxnreg must be in multiples of 8");
-    Check((RegCount >= 24 && RegCount <= 256),
-          "reg_count argument to nvvm.setmaxnreg must be within [24, 256]");
     break;
   }
   case Intrinsic::experimental_convergence_entry:
@@ -6605,14 +6603,6 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
           "llvm.threadlocal.address operand isThreadLocal() must be true");
     break;
   }
-  case Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_cta:
-  case Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_cluster:
-  case Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_gpu:
-  case Intrinsic::nvvm_fence_proxy_tensormap_generic_acquire_sys: {
-    unsigned size = cast<ConstantInt>(Call.getArgOperand(1))->getZExtValue();
-    Check(size == 128, " The only supported value for size operand is 128");
-    break;
-  }
   };
 
   // Verify that there aren't any unmediated control transfers between funclets.
diff --git a/llvm/test/Verifier/NVPTX/fence-proxy.tensormap.ll b/llvm/test/Verifier/NVPTX/fence-proxy.tensormap.ll
new file mode 100644
index 000000000000..4fa7a7ae7100
--- /dev/null
+++ b/llvm/test/Verifier/NVPTX/fence-proxy.tensormap.ll
@@ -0,0 +1,17 @@
+; RUN: not llvm-as %s -o /dev/null 2>&1 | FileCheck %s
+
+define void @test_fence_proxy_tensormap_generic_acquire(ptr addrspace(0) %addr) {
+  ; CHECK: immarg value 127 out of range [128, 129)
+  call void @llvm.nvvm.fence.proxy.tensormap_generic.acquire.cta(ptr addrspace(0) %addr, i32 127);
+
+  ; CHECK: immarg value 129 out of range [128, 129)
+  call void @llvm.nvvm.fence.proxy.tensormap_generic.acquire.cluster(ptr addrspace(0) %addr, i32 129);
+
+  ; CHECK: immarg value 127 out of range [128, 129)
+  call void @llvm.nvvm.fence.proxy.tensormap_generic.acquire.gpu(ptr addrspace(0) %addr, i32 127);
+
+  ; CHECK: immarg value 129 out of range [128, 129)
+  call void @llvm.nvvm.fence.proxy.tensormap_generic.acquire.sys(ptr addrspace(0) %addr, i32 129);
+
+  ret void
+}
diff --git a/llvm/test/Verifier/NVPTX/setmaxnreg.ll b/llvm/test/Verifier/NVPTX/setmaxnreg.ll
index 8999e4ffa667..1afebeab4742 100644
--- a/llvm/test/Verifier/NVPTX/setmaxnreg.ll
+++ b/llvm/test/Verifier/NVPTX/setmaxnreg.ll
@@ -7,8 +7,10 @@ define void @test_set_maxn_reg() {
   ; CHECK: reg_count argument to nvvm.setmaxnreg must be in multiples of 8
   call void @llvm.nvvm.setmaxnreg.inc.sync.aligned.u32(i32 95)
 
-  ; CHECK: reg_count argument to nvvm.setmaxnreg must be within [24, 256]
+  ; CHECK: immarg value 16 out of range [24, 257)
   call void @llvm.nvvm.setmaxnreg.dec.sync.aligned.u32(i32 16)
 
+  ; CHECK: immarg value 264 out of range [24, 257)
+  call void @llvm.nvvm.setmaxnreg.dec.sync.aligned.u32(i32 264)
   ret void
 }
-- 
cgit v1.2.3


From 5875fafdc547889fb089c943a881a9ab6d8a23c0 Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Thu, 19 Jun 2025 10:30:47 +0800
Subject: [X86] Remove CLDEMOTE from Alderlake and later hybrid processors
 (#144662)

SDM doesn't list any hybrid processors in this feature. Besides,
physical machine also reports not supported.
---
 clang/test/Preprocessor/predefined-arch-macros.c | 4 ++--
 llvm/lib/Target/X86/X86.td                       | 2 +-
 llvm/lib/TargetParser/X86TargetParser.cpp        | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index 2d17891071aa..9dfeddbd4d5a 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -2102,7 +2102,7 @@
 // CHECK_ADL_M32: #define __AVX__ 1
 // CHECK_ADL_M32: #define __BMI2__ 1
 // CHECK_ADL_M32: #define __BMI__ 1
-// CHECK_ADL_M32: #define __CLDEMOTE__ 1
+// CHECK_ADL_M32-NOT: #define __CLDEMOTE__ 1
 // CHECK_ADL_M32: #define __CLFLUSHOPT__ 1
 // CHECK_ADL_M32: #define __CLWB__ 1
 // CHECK_ADL_M32: #define __F16C__ 1
@@ -2173,7 +2173,7 @@
 // CHECK_ADL_M64: #define __AVX__ 1
 // CHECK_ADL_M64: #define __BMI2__ 1
 // CHECK_ADL_M64: #define __BMI__ 1
-// CHECK_ADL_M64: #define __CLDEMOTE__ 1
+// CHECK_ADL_M64-NOT: #define __CLDEMOTE__ 1
 // CHECK_ADL_M64: #define __CLFLUSHOPT__ 1
 // CHECK_ADL_M64: #define __CLWB__ 1
 // CHECK_ADL_M64: #define __F16C__ 1
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 2d635835e3ff..b09891652ad9 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -1284,7 +1284,6 @@ def ProcessorFeatures {
                                                   FeatureAVXVNNI,
                                                   FeaturePKU,
                                                   FeatureHRESET,
-                                                  FeatureCLDEMOTE,
                                                   FeatureMOVDIRI,
                                                   FeatureMOVDIR64B,
                                                   FeatureWAITPKG];
@@ -1311,6 +1310,7 @@ def ProcessorFeatures {
                                                   FeatureAVXNECONVERT,
                                                   FeatureENQCMD,
                                                   FeatureUINTR,
+                                                  FeatureCLDEMOTE,
                                                   FeatureAVXVNNIINT8];
   list<SubtargetFeature> SRFFeatures =
     !listconcat(ADLFeatures, SRFAdditionalFeatures);
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 21d05ee389e6..4947b05cd037 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -165,11 +165,11 @@ constexpr FeatureBitset FeaturesAlderlake =
     FeaturesTremont | FeatureADX | FeatureBMI | FeatureBMI2 | FeatureF16C |
     FeatureFMA | FeatureINVPCID | FeatureLZCNT | FeaturePCONFIG | FeaturePKU |
     FeatureSERIALIZE | FeatureSHSTK | FeatureVAES | FeatureVPCLMULQDQ |
-    FeatureCLDEMOTE | FeatureMOVDIR64B | FeatureMOVDIRI | FeatureWAITPKG |
-    FeatureAVXVNNI | FeatureHRESET | FeatureWIDEKL;
+    FeatureMOVDIR64B | FeatureMOVDIRI | FeatureWAITPKG | FeatureAVXVNNI |
+    FeatureHRESET | FeatureWIDEKL;
 constexpr FeatureBitset FeaturesSierraforest =
     FeaturesAlderlake | FeatureCMPCCXADD | FeatureAVXIFMA | FeatureUINTR |
-    FeatureENQCMD | FeatureAVXNECONVERT | FeatureAVXVNNIINT8;
+    FeatureCLDEMOTE | FeatureENQCMD | FeatureAVXNECONVERT | FeatureAVXVNNIINT8;
 constexpr FeatureBitset FeaturesArrowlakeS = FeaturesSierraforest |
     FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 | FeatureSM4;
 constexpr FeatureBitset FeaturesPantherlake =
-- 
cgit v1.2.3


From 351303c28e8feb85c93d8e9480f534653b032735 Mon Sep 17 00:00:00 2001
From: Han-Chung Wang <hanhan0912@gmail.com>
Date: Wed, 18 Jun 2025 20:07:43 -0700
Subject: [mlir][docs] Fix broken links to Traits documentation. (#144820)

---
 mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td | 2 +-
 mlir/include/mlir/IR/BuiltinOps.td               | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 77e3074661ab..481b14cdb462 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -339,7 +339,7 @@ def MemRef_AllocaOp : AllocLikeOp<"alloca", AutomaticAllocationScopeResource,[
     The `alloca` operation allocates memory on the stack, to be automatically
     released when control transfers back from the region of its closest
     surrounding operation with an
-    [`AutomaticAllocationScope`](../Traits.md/#automaticallocationscope) trait.
+    [`AutomaticAllocationScope`](../Traits/#automaticallocationscope) trait.
     The amount of memory allocated is specified by its memref and additional
     operands. For example:
 
diff --git a/mlir/include/mlir/IR/BuiltinOps.td b/mlir/include/mlir/IR/BuiltinOps.td
index 56edd7519cd6..cdc09afe0b67 100644
--- a/mlir/include/mlir/IR/BuiltinOps.td
+++ b/mlir/include/mlir/IR/BuiltinOps.td
@@ -40,7 +40,7 @@ def ModuleOp : Builtin_Op<"module", [
     [graph region](../LangRef.md#control-flow-and-ssacfg-regions) containing a single block
     which can contain any operations and does not have a terminator. Operations
     within this region cannot implicitly capture values defined outside the module,
-    i.e. Modules are [IsolatedFromAbove](../Traits.md#isolatedfromabove). Modules have
+    i.e. Modules are [IsolatedFromAbove](../Traits#isolatedfromabove). Modules have
     an optional [symbol name](../SymbolsAndSymbolTables.md) which can be used to refer
     to them in operations.
 
-- 
cgit v1.2.3


From 7b989ade35a43357f9152198ee2c76899df9a56d Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Wed, 18 Jun 2025 22:49:21 -0700
Subject: [lldb/crashlog] Make interactive mode the new default (#144839)

This patch makes interactive mode as the default when using the crashlog
command. It replaces the existing `-i|--interactive` flag with a new
`-m|--mode` option, that can either be `interactive` or `batch`.

By default, when the option is not explicitely set by the user, the
interactive mode is selected, however, lldb will fallback to batch mode
if the command interpreter is not interactive or if stdout is not a tty.

This also adds some railguards to prevent users from using interactive
only options with the batch mode and updates the tests accordingly.

rdar://97801509

Differential Revision: https://reviews.llvm.org/D141658

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py                   | 126 +++++++++++++--------
 .../Python/Crashlog/altered_threadState.test       |   2 +-
 .../ScriptInterpreter/Python/Crashlog/json.test    |   6 +-
 .../Python/Crashlog/no_threadState.test            |   2 +-
 .../skipped_status_interactive_crashlog.test       |   2 +-
 .../ScriptInterpreter/Python/Crashlog/text.test    |   2 +-
 6 files changed, 85 insertions(+), 55 deletions(-)

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index 6615c3353ffe..5f07cda2892a 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -31,6 +31,7 @@ import argparse
 import concurrent.futures
 import contextlib
 import datetime
+import enum
 import json
 import os
 import platform
@@ -45,7 +46,6 @@ import threading
 import time
 import uuid
 
-
 print_lock = threading.RLock()
 
 try:
@@ -1582,9 +1582,12 @@ def load_crashlog_in_scripted_process(debugger, crashlog_path, options, result):
                 debugger.RunCommandInterpreter(True, False, run_options, 0, False, True)
 
 
-def CreateSymbolicateCrashLogOptions(
-    command_name, description, add_interactive_options
-):
+class CrashLogLoadingMode(str, enum.Enum):
+    batch = "batch"
+    interactive = "interactive"
+
+
+def CreateSymbolicateCrashLogOptions(command_name, description):
     usage = "crashlog [options] <FILE> [FILE ...]"
     arg_parser = argparse.ArgumentParser(
         description=description,
@@ -1600,6 +1603,12 @@ def CreateSymbolicateCrashLogOptions(
         help="crash report(s) to symbolicate",
     )
 
+    arg_parser.add_argument(
+        "-m",
+        "--mode",
+        choices=[mode.value for mode in CrashLogLoadingMode],
+        help="change how the symbolicated process and threads are displayed to the user (default: interactive)",
+    )
     arg_parser.add_argument(
         "--version",
         "-V",
@@ -1736,36 +1745,35 @@ def CreateSymbolicateCrashLogOptions(
         help=argparse.SUPPRESS,
         default=False,
     )
-    if add_interactive_options:
-        arg_parser.add_argument(
-            "-i",
-            "--interactive",
-            action="store_true",
-            help="parse a crash log and load it in a ScriptedProcess",
-            default=False,
-        )
-        arg_parser.add_argument(
-            "-b",
-            "--batch",
-            action="store_true",
-            help="dump symbolicated stackframes without creating a debug session",
-            default=True,
-        )
-        arg_parser.add_argument(
-            "--target",
-            "-t",
-            dest="target_path",
-            help="the target binary path that should be used for interactive crashlog (optional)",
-            default=None,
-        )
-        arg_parser.add_argument(
-            "--skip-status",
-            "-s",
-            dest="skip_status",
-            action="store_true",
-            help="prevent the interactive crashlog to dump the process status and thread backtrace at launch",
-            default=False,
-        )
+    arg_parser.add_argument(
+        "--target",
+        "-t",
+        dest="target_path",
+        help="the target binary path that should be used for interactive crashlog (optional)",
+        default=None,
+    )
+    arg_parser.add_argument(
+        "--skip-status",
+        "-s",
+        dest="skip_status",
+        action="store_true",
+        help="prevent the interactive crashlog to dump the process status and thread backtrace at launch",
+        default=False,
+    )
+    legacy_group = arg_parser.add_mutually_exclusive_group()
+    legacy_group.add_argument(
+        "-i",
+        "--interactive",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+    legacy_group.add_argument(
+        "-b",
+        "--batch",
+        action="store_true",
+        help=argparse.SUPPRESS,
+    )
+
     return arg_parser
 
 
@@ -1778,7 +1786,7 @@ for use at the LLDB command line. After a crash log has been parsed and symbolic
 created that has all of the shared libraries loaded at the load addresses found in the crash log file. This allows
 you to explore the program as if it were stopped at the locations described in the crash log and functions can
 be disassembled and lookups can be performed using the addresses found in the crash log."""
-    return CreateSymbolicateCrashLogOptions("crashlog", description, True)
+    return CreateSymbolicateCrashLogOptions("crashlog", description)
 
 
 def SymbolicateCrashLogs(debugger, command_args, result, is_command):
@@ -1794,8 +1802,35 @@ def SymbolicateCrashLogs(debugger, command_args, result, is_command):
         result.SetError(str(e))
         return
 
+    # To avoid breaking existing users, we should keep supporting legacy flags
+    # even if we don't use them / advertise them anymore.
+    if not options.mode:
+        if options.batch:
+            options.mode = CrashLogLoadingMode.batch
+        else:
+            options.mode = CrashLogLoadingMode.interactive
+
+    if options.mode != CrashLogLoadingMode.interactive and (
+        options.target_path or options.skip_status
+    ):
+        print(
+            "Target path (-t) and skipping process status (-s) options can only used in interactive mode (-m=interactive)."
+        )
+        print("Aborting symbolication.")
+        arg_parser.print_help()
+        return
+
+    if options.version:
+        print(debugger.GetVersionString())
+        return
+
+    if options.debug:
+        print("command_args = %s" % command_args)
+        print("options", options)
+        print("args", options.reports)
+
     # Interactive mode requires running the crashlog command from inside lldb.
-    if options.interactive and not is_command:
+    if options.mode == CrashLogLoadingMode.interactive and not is_command:
         lldb_exec = (
             subprocess.check_output(["/usr/bin/xcrun", "-f", "lldb"])
             .decode("utf-8")
@@ -1821,31 +1856,26 @@ def SymbolicateCrashLogs(debugger, command_args, result, is_command):
         print(debugger.GetVersionString())
         return
 
-    if options.debug:
-        print("command_args = %s" % command_args)
-        print("options", options)
-        print("args", options.reports)
-
     if options.debug_delay > 0:
         print("Waiting %u seconds for debugger to attach..." % options.debug_delay)
         time.sleep(options.debug_delay)
     error = lldb.SBError()
 
     def should_run_in_interactive_mode(options, ci):
-        if options.interactive:
-            return True
-        elif options.batch:
+        if options.mode == CrashLogLoadingMode.batch:
             return False
-        # elif ci and ci.IsInteractive():
-        #     return True
+        elif options.mode == CrashLogLoadingMode.interactive or (
+            ci and ci.IsInteractive()
+        ):
+            return True
         else:
-            return False
+            return sys.stdout.isatty()
 
     ci = debugger.GetCommandInterpreter()
 
     if options.reports:
         for crashlog_file in options.reports:
-            crashlog_path = os.path.expanduser(crashlog_file)
+            crashlog_path = os.path.normpath(os.path.expanduser(crashlog_file))
             if not os.path.exists(crashlog_path):
                 raise FileNotFoundError(
                     "crashlog file %s does not exist" % crashlog_path
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test
index 5a946a38b195..d925324822de 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/altered_threadState.test
@@ -1,7 +1,7 @@
 # RUN: %clang_host -g %S/Inputs/test.c -o %t.out
 # RUN: cp %S/Inputs/altered_threadState.crash %t.crash
 # RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.crash --offsets '{"main":20, "bar":9, "foo":16}'
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog -b %t.crash' 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
 
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/json.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/json.test
index c2e23e82be7f..d5c6d915316e 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/json.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/json.test
@@ -2,12 +2,12 @@
 
 # RUN: cp %S/Inputs/a.out.ips %t.crash
 # RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.crash --offsets '{"main":20, "bar":9, "foo":16}' --json
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.crash' 2>&1 | FileCheck %s
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog -c %t.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog --mode batch %t.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog --mode batch -c %t.crash' 2>&1 | FileCheck %s
 
 # RUN: cp %S/Inputs/a.out.ips %t.nometadata.crash
 # RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.nometadata.crash --offsets '{"main":20, "bar":9, "foo":16}' --json --no-metadata
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.nometadata.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog --mode batch %t.nometadata.crash' 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
 
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/no_threadState.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/no_threadState.test
index 5b5cef40716c..2e4b46c8c240 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/no_threadState.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/no_threadState.test
@@ -2,7 +2,7 @@
 
 # RUN: cp %S/Inputs/no_threadState.ips %t.crash
 # RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.crash --offsets '{"main":20, "bar":9, "foo":16}' --json
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog --mode batch %t.crash' 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
 
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
index 64cd0904371a..52a185b8cf76 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/skipped_status_interactive_crashlog.test
@@ -3,7 +3,7 @@
 # RUN: mkdir -p %t.dir
 # RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test
 # RUN: %lldb -b -o 'command script import lldb.macosx.crashlog' \
-# RUN: -o 'crashlog -a -i -s -t %t.dir/multithread-test %S/Inputs/interactive_crashlog/multithread-test.ips' \
+# RUN: -o 'crashlog -a -s -t %t.dir/multithread-test %S/Inputs/interactive_crashlog/multithread-test.ips' \
 # RUN: -o 'command source -s 0 %s' 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/text.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/text.test
index e9d1c5e98fb3..eec30a1da64c 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/text.test
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/text.test
@@ -1,7 +1,7 @@
 # RUN: %clang_host -g %S/Inputs/test.c -o %t.out
 # RUN: cp %S/Inputs/a.out.crash %t.crash
 # RUN: %python %S/patch-crashlog.py --binary %t.out --crashlog %t.crash --offsets '{"main":20, "bar":9, "foo":16}'
-# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog %t.crash' 2>&1 | FileCheck %s
+# RUN: %lldb %t.out -o 'command script import lldb.macosx.crashlog' -o 'crashlog -b %t.crash' 2>&1 | FileCheck %s
 
 # CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
 
-- 
cgit v1.2.3


From 590066bee70db37636311881c5b232464d6d4aec Mon Sep 17 00:00:00 2001
From: Rajat Bajpai <rbajpai@nvidia.com>
Date: Thu, 19 Jun 2025 12:18:17 +0530
Subject: [NVPTX] Add family-specific architectures support (#141899)

This change adds family-specific architecture variants support added in [PTX ISA
8.8](https://docs.nvidia.com/cuda/parallel-thread-execution/#ptx-isa-version-8-8).
These architecture variants have "f" suffix. For example, sm_100f.

This change doesn't promote existing features to family-specific
architecture.
---
 llvm/docs/NVPTXUsage.rst                | 52 +++++++++++++++++++++-
 llvm/lib/Target/NVPTX/NVPTX.td          | 76 ++++++++++++++++++++++++++++-----
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td |  8 ++--
 llvm/lib/Target/NVPTX/NVPTXSubtarget.h  | 42 ++++++++++++------
 llvm/test/CodeGen/NVPTX/sm-version.ll   | 20 +++++++++
 5 files changed, 169 insertions(+), 29 deletions(-)

diff --git a/llvm/docs/NVPTXUsage.rst b/llvm/docs/NVPTXUsage.rst
index abd7ca545364..11017fe4e01b 100644
--- a/llvm/docs/NVPTXUsage.rst
+++ b/llvm/docs/NVPTXUsage.rst
@@ -147,7 +147,57 @@ Example: 32-bit PTX for CUDA Driver API: ``nvptx-nvidia-cuda``
 
 Example: 64-bit PTX for CUDA Driver API: ``nvptx64-nvidia-cuda``
 
-
+.. _nvptx_arch_hierarchy:
+
+NVPTX Architecture Hierarchy and Ordering
+=========================================
+
+GPU architectures: sm_2Y/sm_3Y/sm_5Y/sm_6Y/sm_7Y/sm_8Y/sm_9Y/sm_10Y/sm_12Y
+('Y' represents version within the architecture)
+The architectures have name of form ``sm_XYz`` where ``X`` represent the generation
+number, ``Y`` represents the version within the architecture, and ``z`` represents
+the optional feature suffix.
+If ``X1Y1 <= X2Y2``, then GPU capabilities of ``sm_X1Y1`` are included in ``sm_X2Y2``.
+For example, take ``sm_90`` (9 represents ``X``, 0 represents ``Y``, and no feature
+suffix) and ``sm_103`` architectures (10 represents ``X``, 3 represents ``Y``, and no
+feature suffix). Since 90 <= 103, ``sm_90`` is compatible with ``sm_103``.
+
+The family-specific variants have ``f`` feature suffix and they follow
+following order:
+``sm_X{Y2}f > sm_X{Y1}f`` iff ``Y2 > Y1``
+``sm_XY{f} > sm_{XY}{}``
+
+For example, take ``sm_100f`` (10 represents ``X``, 0 represents ``Y``, and ``f``
+represents ``z``) and ``sm_103f`` (10 represents ``X``, 3 represents ``Y``, and ``f``
+represents ``z``) architecture variants. Since ``Y1 < Y2``, ``sm_100f`` is compatible with
+``sm_103f``. Similarly based on the second rule, ``sm_90`` is compatible with ``sm_103f``.
+
+Some counter examples, take ``sm_100f`` and ``sm_120f`` (12 represents ``X``, 0
+represents ``Y``, and ``f`` represents ``z``) architecture variants. Since both
+belongs to different family i.e. ``X1 != X2``, ``sm_100f`` is not compatible with
+``sm_120f``.
+
+The architecture-specific variants have ``a`` feature suffix and they follow
+following order:
+``sm_XY{a} > sm_XY{f} > sm_{XY}{}``
+
+For example, take ``sm_103a`` (10 represents ``X``, 3 represents ``Y``, and ``a``
+represents ``z``), ``sm_103f``, and ``sm_103`` architecture variants. The ``sm_103`` is
+compatible with ``sm_103a`` and ``sm_103f``, and ``sm_103f`` is compatible with ``sm_103a``.
+
+Encoding := Arch * 10 + 2 (for 'f') + 1 (for 'a')
+Arch := X * 10 + Y
+
+For example, ``sm_103f`` is encoded as 1032 (103 * 10 + 2) and ``sm_103a`` is
+encoded as 1033 (103 * 10 + 2 + 1).
+
+This encoding allows simple partial ordering of the architectures.
+
+* Compare Family and Arch by dividing FullSMVersion by 100 and 10
+  respectively before the comparison.
+* Compare within the family by comparing FullSMVersion, given both belongs to
+  the same family.
+* Detect ``a`` variants by checking FullSMVersion & 1.
 
 .. _nvptx_intrinsics:
 
diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td
index ff9a187ecf72..83992606bc41 100644
--- a/llvm/lib/Target/NVPTX/NVPTX.td
+++ b/llvm/lib/Target/NVPTX/NVPTX.td
@@ -33,20 +33,69 @@ class FeaturePTX<int version>:
    SubtargetFeature<"ptx"# version, "PTXVersion",
                     "" # version,
                     "Use PTX version " # version>;
-
+// NVPTX Architecture Hierarchy and Ordering:
+//
+// GPU architectures: sm_2Y/sm_3Y/sm_5Y/sm_6Y/sm_7Y/sm_8Y/sm_9Y/sm_10Y/sm_12Y
+// ('Y' represents version within the architecture)
+// The architectures have name of form sm_XYz where 'X' represent the generation
+// number, 'Y' represents the version within the architecture, and 'z' represents
+// the optional feature suffix.
+// If X1Y1 <= X2Y2, then GPU capabilities of sm_X1Y1 are included in sm_X2Y2.
+// For example, take sm_90 (9 represents 'X', 0 represents 'Y', and no feature
+// suffix) and sm_103 architectures (10 represents 'X', 3 represents 'Y', and no
+// feature suffix). Since 90 <= 103, sm_90 is compatible with sm_103.
+//
+// The family-specific variants have 'f' feature suffix and they follow
+// following order:
+// sm_X{Y2}f > sm_X{Y1}f iff Y2 > Y1
+// sm_XY{f} > sm_{XY}{}
+//
+// For example, take sm_100f (10 represents 'X', 0 represents 'Y', and 'f'
+// represents 'z') and sm_103f (10 represents 'X', 3 represents 'Y', and 'f'
+// represents 'z') architecture variants. Since Y1 < Y2, sm_100f is compatible with
+// sm_103f. Similarly based on the second rule, sm_90 is compatible with sm_103f.
+//
+// Some counter examples, take sm_100f and sm_120f (12 represents 'X', 0
+// represents 'Y', and 'f' represents 'z') architecture variants. Since both
+// belongs to different family i.e. X1 != X2, sm_100f is not compatible with
+// sm_120f.
+//
+// The architecture-specific variants have 'a' feature suffix and they follow
+// following order:
+// sm_XY{a} > sm_XY{f} > sm_{XY}{}
+//
+// For example, take sm_103a (10 represents 'X', 3 represents 'Y', and 'a'
+// represents 'z'), sm_103f, and sm_103 architecture variants. The sm_103 is
+// compatible with sm_103a and sm_103f, and sm_103f is compatible with sm_103a.
+//
+// Encoding := Arch * 10 + 2 (for 'f') + 1 (for 'a')
+// Arch := X * 10 + Y
+//
+// For example, sm_103a is encoded as 1033 (103 * 10 + 2 + 1) and sm_103f is
+// encoded as 1032 (103 * 10 + 2).
+//
+// This encoding allows simple partial ordering of the architectures.
+//  + Compare Family and Arch by dividing FullSMVersion by 100 and 10
+//    respectively before the comparison.
+//  + Compare within the family by comparing FullSMVersion, given both belongs to
+//    the same family.
+//  + Detect 'a' variants by checking FullSMVersion & 1.
 foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53,
               60, 61, 62, 70, 72, 75, 80, 86, 87,
-              89, 90, 100, 101, 103, 120, 121] in
-  def SM#sm: FeatureSM<""#sm, !mul(sm, 10)>;
+              89, 90, 100, 101, 103, 120, 121] in {
+  // Base SM version (e.g. FullSMVersion for sm_100 is 1000)
+  def SM#sm : FeatureSM<""#sm, !mul(sm, 10)>;
+
+  // Family-specific targets which are compatible within same family
+  // (e.g. FullSMVersion for sm_100f is 1002)
+  if !ge(sm, 100) then
+    def SM#sm#f : FeatureSM<""#sm#"f", !add(!mul(sm, 10), 2)>;
 
-// Arch-specific targets. PTX for these is not compatible with any other
-// architectures.
-def SM90a : FeatureSM<"90a", 901>;
-def SM100a: FeatureSM<"100a", 1001>;
-def SM101a: FeatureSM<"101a", 1011>;
-def SM103a: FeatureSM<"103a", 1031>;
-def SM120a: FeatureSM<"120a", 1201>;
-def SM121a: FeatureSM<"121a", 1211>;
+  // Architecture-specific targets which are incompatible across architectures
+  // (e.g. FullSMVersion for sm_100a is 1003)
+  if !ge(sm, 90) then
+    def SM#sm#a : FeatureSM<""#sm#"a", !add(!mul(sm, 10), 3)>;
+}
 
 foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 62, 63, 64, 65,
                    70, 71, 72, 73, 74, 75, 76, 77, 78,
@@ -83,14 +132,19 @@ def : Proc<"sm_90",   [SM90, PTX78]>;
 def : Proc<"sm_90a",  [SM90a, PTX80]>;
 def : Proc<"sm_100",  [SM100, PTX86]>;
 def : Proc<"sm_100a", [SM100a, PTX86]>;
+def : Proc<"sm_100f", [SM100f, PTX88]>;
 def : Proc<"sm_101",  [SM101, PTX86]>;
 def : Proc<"sm_101a", [SM101a, PTX86]>;
+def : Proc<"sm_101f", [SM101f, PTX88]>;
 def : Proc<"sm_103",  [SM103, PTX88]>;
 def : Proc<"sm_103a", [SM103a, PTX88]>;
+def : Proc<"sm_103f", [SM103f, PTX88]>;
 def : Proc<"sm_120",  [SM120, PTX87]>;
 def : Proc<"sm_120a", [SM120a, PTX87]>;
+def : Proc<"sm_120f", [SM120f, PTX88]>;
 def : Proc<"sm_121",  [SM121, PTX88]>;
 def : Proc<"sm_121a", [SM121a, PTX88]>;
+def : Proc<"sm_121f", [SM121f, PTX88]>;
 
 def NVPTXInstrInfo : InstrInfo {
 }
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 5dbdce52f055..bbe99dec5c44 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -158,10 +158,10 @@ class hasPTX<int version>: Predicate<"Subtarget->getPTXVersion() >= " # version>
 class hasSM<int version>: Predicate<"Subtarget->getSmVersion() >= " # version>;
 
 // Explicit records for arch-accelerated SM versions
-def hasSM90a : Predicate<"Subtarget->getFullSmVersion() == 901">;
-def hasSM100a : Predicate<"Subtarget->getFullSmVersion() == 1001">;
-def hasSM101a : Predicate<"Subtarget->getFullSmVersion() == 1011">;
-def hasSM120a : Predicate<"Subtarget->getFullSmVersion() == 1201">;
+def hasSM90a : Predicate<"Subtarget->getSmVersion() == 90 && Subtarget->hasArchAccelFeatures()">;
+def hasSM100a : Predicate<"Subtarget->getSmVersion() == 100 && Subtarget->hasArchAccelFeatures()">;
+def hasSM101a : Predicate<"Subtarget->getSmVersion() == 101 && Subtarget->hasArchAccelFeatures()">;
+def hasSM120a : Predicate<"Subtarget->getSmVersion() == 120 && Subtarget->hasArchAccelFeatures()">;
 
 // non-sync shfl instructions are not available on sm_70+ in PTX6.4+
 def hasSHFL : Predicate<"!(Subtarget->getSmVersion() >= 70"
diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
index d2eae4882682..8810feaee297 100644
--- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -108,8 +108,8 @@ public:
     switch (FullSmVersion) {
     default:
       break;
-    case 1001: // sm_100a
-    case 1011: // sm_101a
+    case 1003: // sm_100a
+    case 1013: // sm_101a
       HasTcgen05 = true;
       break;
     }
@@ -120,9 +120,15 @@ public:
   // TMA G2S copy with cta_group::1/2 support
   bool hasCpAsyncBulkTensorCTAGroupSupport() const {
     // TODO: Update/tidy-up after the family-conditional support arrives
-    return ((FullSmVersion == 1001 || FullSmVersion == 1011) &&
-            PTXVersion >= 86) ||
-           (FullSmVersion == 1031 && PTXVersion >= 88);
+    switch (FullSmVersion) {
+    case 1003:
+    case 1013:
+      return PTXVersion >= 86;
+    case 1033:
+      return PTXVersion >= 88;
+    default:
+      return false;
+    }
   }
 
   // Prior to CUDA 12.3 ptxas did not recognize that the trap instruction
@@ -136,14 +142,24 @@ public:
   bool hasCvtaParam() const { return SmVersion >= 70 && PTXVersion >= 77; }
   unsigned int getFullSmVersion() const { return FullSmVersion; }
   unsigned int getSmVersion() const { return getFullSmVersion() / 10; }
-  // GPUs with "a" suffix have include architecture-accelerated features that
-  // are supported on the specified architecture only, hence such targets do not
-  // follow the onion layer model. hasArchAccelFeatures() allows
-  // distinguishing such GPU variants from the base GPU architecture.
-  // - 0 represents base GPU model,
-  // - non-zero value identifies particular architecture-accelerated variant.
-  bool hasArchAccelFeatures() const { return getFullSmVersion() % 10; }
-
+  // GPUs with "a" suffix have architecture-accelerated features that are
+  // supported on the specified architecture only, hence such targets do not
+  // follow the onion layer model. hasArchAccelFeatures() allows distinguishing
+  // such GPU variants from the base GPU architecture.
+  // - false represents non-accelerated architecture.
+  // - true represents architecture-accelerated variant.
+  bool hasArchAccelFeatures() const {
+    return (getFullSmVersion() & 1) && PTXVersion >= 80;
+  }
+  // GPUs with 'f' suffix have architecture-accelerated features which are
+  // portable across all future architectures under same SM major. For example,
+  // sm_100f features will work for sm_10X*f*/sm_10X*a* future architectures.
+  // - false represents non-family-specific architecture.
+  // - true represents family-specific variant.
+  bool hasFamilySpecificFeatures() const {
+    return getFullSmVersion() % 10 == 2 ? PTXVersion >= 88
+                                        : hasArchAccelFeatures();
+  }
   // If the user did not provide a target we default to the `sm_30` target.
   std::string getTargetName() const {
     return TargetName.empty() ? "sm_30" : TargetName;
diff --git a/llvm/test/CodeGen/NVPTX/sm-version.ll b/llvm/test/CodeGen/NVPTX/sm-version.ll
index 9705a2f3ba73..3a154a1b9ac9 100644
--- a/llvm/test/CodeGen/NVPTX/sm-version.ll
+++ b/llvm/test/CodeGen/NVPTX/sm-version.ll
@@ -18,14 +18,19 @@
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_90a | FileCheck %s --check-prefix=SM90a
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_100 | FileCheck %s --check-prefix=SM100
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_100a | FileCheck %s --check-prefix=SM100a
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_100f | FileCheck %s --check-prefix=SM100f
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_101 | FileCheck %s --check-prefix=SM101
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_101a | FileCheck %s --check-prefix=SM101a
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_101f | FileCheck %s --check-prefix=SM101f
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_103 | FileCheck %s --check-prefix=SM103
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_103a | FileCheck %s --check-prefix=SM103a
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_103f | FileCheck %s --check-prefix=SM103f
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_120 | FileCheck %s --check-prefix=SM120
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_120a | FileCheck %s --check-prefix=SM120a
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_120f | FileCheck %s --check-prefix=SM120f
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_121 | FileCheck %s --check-prefix=SM121
 ; RUN: llc < %s -mtriple=nvptx -mcpu=sm_121a | FileCheck %s --check-prefix=SM121a
+; RUN: llc < %s -mtriple=nvptx -mcpu=sm_121f | FileCheck %s --check-prefix=SM121f
 
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_20 | FileCheck %s --check-prefix=SM20
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_21 | FileCheck %s --check-prefix=SM21
@@ -47,14 +52,19 @@
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_90a | FileCheck %s --check-prefix=SM90a
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100 | FileCheck %s --check-prefix=SM100
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a | FileCheck %s --check-prefix=SM100a
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100f | FileCheck %s --check-prefix=SM100f
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_101 | FileCheck %s --check-prefix=SM101
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_101a | FileCheck %s --check-prefix=SM101a
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_101f | FileCheck %s --check-prefix=SM101f
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_103 | FileCheck %s --check-prefix=SM103
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_103a | FileCheck %s --check-prefix=SM103a
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_103f | FileCheck %s --check-prefix=SM103f
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_120 | FileCheck %s --check-prefix=SM120
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_120a | FileCheck %s --check-prefix=SM120a
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_120f | FileCheck %s --check-prefix=SM120f
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_121 | FileCheck %s --check-prefix=SM121
 ; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_121a | FileCheck %s --check-prefix=SM121a
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_121f | FileCheck %s --check-prefix=SM121f
 
 ; SM20: .version 3.2
 ; SM21: .version 3.2
@@ -76,14 +86,19 @@
 ; SM90a: .version 8.0
 ; SM100: .version 8.6
 ; SM100a: .version 8.6
+; SM100f: .version 8.8
 ; SM101: .version 8.6
 ; SM101a: .version 8.6
+; SM101f: .version 8.8
 ; SM103: .version 8.8
 ; SM103a: .version 8.8
+; SM103f: .version 8.8
 ; SM120: .version 8.7
 ; SM120a: .version 8.7
+; SM120f: .version 8.8
 ; SM121: .version 8.8
 ; SM121a: .version 8.8
+; SM121f: .version 8.8
 
 ; SM20: .target sm_20
 ; SM21: .target sm_21
@@ -105,11 +120,16 @@
 ; SM90a: .target sm_90a
 ; SM100: .target sm_100
 ; SM100a: .target sm_100a
+; SM100f: .target sm_100f
 ; SM101: .target sm_101
 ; SM101a: .target sm_101a
+; SM101f: .target sm_101f
 ; SM103: .target sm_103
 ; SM103a: .target sm_103a
+; SM103f: .target sm_103f
 ; SM120: .target sm_120
 ; SM120a: .target sm_120a
+; SM120f: .target sm_120f
 ; SM121: .target sm_121
 ; SM121a: .target sm_121a
+; SM121f: .target sm_121f
-- 
cgit v1.2.3


From 03461c9c6e21e43a6e1c699bfb254ddb3d575c93 Mon Sep 17 00:00:00 2001
From: Hsiangkai Wang <hsiangkai.wang@arm.com>
Date: Thu, 19 Jun 2025 07:56:30 +0100
Subject: [mlir][gpu][spirv] Remove rotation semantics of gpu.shuffle up/down
 (#139105)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

From the description of gpu.shuffle operation, shuffle up/down rotates
values in the subgroup because it applies modulo on the shifted value to
calculate the result lane ID. It is inconsistent with the definition of
SPIR-V shuffle up/down and NVVM data movement definitions within
subgroup.

In NVVM, it says

"If the computed source lane index j is in range, the returned i32 value
will be the value of %a from lane j; otherwise, it will be the the value
of %a from the current thread."

It will keep the original value if the result land ID is out of range.

In SPIR-V OpGroupNonUniformShuffleUp and OpGroupNonUniformShuffleDown,
it says

"The resulting value is undefined if Delta is greater than the current
invocation’s id within the scope or if the identified invocation is not
in scope restricted tangle."

It's an undefined value if the result land ID is out of range.

Anyway, there is no circular movement in shuffle up/down from these 2
specifications. This patch removes the circular movement in gpu.shuffle
up/down and lower gpu.shuffle up/down to SPIR-V
OpGroupNonUniformShuffleUp and OpGroupNonUniformShuffleDown directly.

Reference:

https://docs.nvidia.com/cuda/archive/12.2.1/nvvm-ir-spec/index.html#data-movement

https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpGroupNonUniformShuffleUp

https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpGroupNonUniformShuffleDown
---
 mlir/include/mlir/Dialect/GPU/IR/GPUOps.td    |  6 ++-
 mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp | 45 ++++++++++++++---
 mlir/test/Conversion/GPUToSPIRV/shuffle.mlir  | 71 ++++++++++++++++++++++++++-
 3 files changed, 111 insertions(+), 11 deletions(-)

diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index 15b14c767b66..a81b2e83ddef 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1332,7 +1332,8 @@ def GPU_ShuffleOp : GPU_Op<
     %3, %4 = gpu.shuffle down %0, %cst1, %width : f32
     ```
 
-    For lane `k`, returns the value from lane `(k + 1) % width`.
+    For lane `k`, returns the value from lane `(k + cst1)`. If `(k + cst1)` is
+    bigger than or equal to `width`, the value is poison and `valid` is `false`.
 
     `up` example:
 
@@ -1341,7 +1342,8 @@ def GPU_ShuffleOp : GPU_Op<
     %5, %6 = gpu.shuffle up %0, %cst1, %width : f32
     ```
 
-    For lane `k`, returns the value from lane `(k - 1) % width`.
+    For lane `k`, returns the value from lane `(k - cst1)`. If `(k - cst1)` is
+    smaller than `0`, the value is poison and `valid` is `false`.
 
     `idx` example:
 
diff --git a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
index 46db5d3fdca3..93c76d267c51 100644
--- a/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
+++ b/mlir/lib/Conversion/GPUToSPIRV/GPUToSPIRV.cpp
@@ -435,26 +435,57 @@ LogicalResult GPUShuffleConversion::matchAndRewrite(
     return rewriter.notifyMatchFailure(
         shuffleOp, "shuffle width and target subgroup size mismatch");
 
+  assert(!adaptor.getOffset().getType().isSignedInteger() &&
+         "shuffle offset must be a signless/unsigned integer");
+
   Location loc = shuffleOp.getLoc();
-  Value trueVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
-                                            shuffleOp.getLoc(), rewriter);
   auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
   Value result;
+  Value validVal;
 
   switch (shuffleOp.getMode()) {
-  case gpu::ShuffleMode::XOR:
+  case gpu::ShuffleMode::XOR: {
     result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
         loc, scope, adaptor.getValue(), adaptor.getOffset());
+    validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
+                                         shuffleOp.getLoc(), rewriter);
     break;
-  case gpu::ShuffleMode::IDX:
+  }
+  case gpu::ShuffleMode::IDX: {
     result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
         loc, scope, adaptor.getValue(), adaptor.getOffset());
+    validVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
+                                         shuffleOp.getLoc(), rewriter);
+    break;
+  }
+  case gpu::ShuffleMode::DOWN: {
+    result = rewriter.create<spirv::GroupNonUniformShuffleDownOp>(
+        loc, scope, adaptor.getValue(), adaptor.getOffset());
+
+    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value resultLaneId =
+        rewriter.create<arith::AddIOp>(loc, laneId, adaptor.getOffset());
+    validVal = rewriter.create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
+                                              resultLaneId, adaptor.getWidth());
     break;
-  default:
-    return rewriter.notifyMatchFailure(shuffleOp, "unimplemented shuffle mode");
+  }
+  case gpu::ShuffleMode::UP: {
+    result = rewriter.create<spirv::GroupNonUniformShuffleUpOp>(
+        loc, scope, adaptor.getValue(), adaptor.getOffset());
+
+    Value laneId = rewriter.create<gpu::LaneIdOp>(loc, widthAttr);
+    Value resultLaneId =
+        rewriter.create<arith::SubIOp>(loc, laneId, adaptor.getOffset());
+    auto i32Type = rewriter.getIntegerType(32);
+    validVal = rewriter.create<arith::CmpIOp>(
+        loc, arith::CmpIPredicate::sge, resultLaneId,
+        rewriter.create<arith::ConstantOp>(
+            loc, i32Type, rewriter.getIntegerAttr(i32Type, 0)));
+    break;
+  }
   }
 
-  rewriter.replaceOp(shuffleOp, {result, trueVal});
+  rewriter.replaceOp(shuffleOp, {result, validVal});
   return success();
 }
 
diff --git a/mlir/test/Conversion/GPUToSPIRV/shuffle.mlir b/mlir/test/Conversion/GPUToSPIRV/shuffle.mlir
index d3d8ec0dab40..e93f69704f25 100644
--- a/mlir/test/Conversion/GPUToSPIRV/shuffle.mlir
+++ b/mlir/test/Conversion/GPUToSPIRV/shuffle.mlir
@@ -15,8 +15,8 @@ gpu.module @kernels {
 
     // CHECK: %[[MASK:.+]] = spirv.Constant 8 : i32
     // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
-    // CHECK: %{{.+}} = spirv.Constant true
     // CHECK: %{{.+}} = spirv.GroupNonUniformShuffleXor <Subgroup> %[[VAL]], %[[MASK]] : f32, i32
+    // CHECK: %{{.+}} = spirv.Constant true
     %result, %valid = gpu.shuffle xor %val, %mask, %width : f32
     gpu.return
   }
@@ -64,11 +64,78 @@ gpu.module @kernels {
 
     // CHECK: %[[MASK:.+]] = spirv.Constant 8 : i32
     // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
-    // CHECK: %{{.+}} = spirv.Constant true
     // CHECK: %{{.+}} = spirv.GroupNonUniformShuffle <Subgroup> %[[VAL]], %[[MASK]] : f32, i32
+    // CHECK: %{{.+}} = spirv.Constant true
     %result, %valid = gpu.shuffle idx %val, %mask, %width : f32
     gpu.return
   }
 }
 
 }
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformShuffle, GroupNonUniformShuffleRelative], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @shuffle_down()
+  gpu.func @shuffle_down() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 16 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 16 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformShuffleDown <Subgroup> %[[VAL]], %[[OFFSET]] : f32, i32
+
+    // CHECK: %[[INVOCATION_ID_ADDR:.+]] = spirv.mlir.addressof @__builtin__SubgroupLocalInvocationId__ : !spirv.ptr<i32, Input>
+    // CHECK: %[[LANE_ID:.+]] = spirv.Load "Input" %[[INVOCATION_ID_ADDR]] : i32
+    // CHECK: %[[VAL_LANE_ID:.+]] = spirv.IAdd %[[LANE_ID]], %[[OFFSET]] : i32
+    // CHECK: %[[VALID:.+]] = spirv.ULessThan %[[VAL_LANE_ID]], %[[WIDTH]] : i32
+
+    %result, %valid = gpu.shuffle down %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
+
+// -----
+
+module attributes {
+  gpu.container_module,
+  spirv.target_env = #spirv.target_env<#spirv.vce<v1.4, [Shader, GroupNonUniformShuffle, GroupNonUniformShuffleRelative], []>,
+    #spirv.resource_limits<subgroup_size = 16>>
+} {
+
+gpu.module @kernels {
+  // CHECK-LABEL:  spirv.func @shuffle_up()
+  gpu.func @shuffle_up() kernel
+    attributes {spirv.entry_point_abi = #spirv.entry_point_abi<workgroup_size = [16, 1, 1]>} {
+    %offset = arith.constant 4 : i32
+    %width = arith.constant 16 : i32
+    %val = arith.constant 42.0 : f32
+
+    // CHECK: %[[OFFSET:.+]] = spirv.Constant 4 : i32
+    // CHECK: %[[WIDTH:.+]] = spirv.Constant 16 : i32
+    // CHECK: %[[VAL:.+]] = spirv.Constant 4.200000e+01 : f32
+    // CHECK: %{{.+}} = spirv.GroupNonUniformShuffleUp <Subgroup> %[[VAL]], %[[OFFSET]] : f32, i32
+
+    // CHECK: %[[INVOCATION_ID_ADDR:.+]] = spirv.mlir.addressof @__builtin__SubgroupLocalInvocationId__ : !spirv.ptr<i32, Input>
+    // CHECK: %[[LANE_ID:.+]] = spirv.Load "Input" %[[INVOCATION_ID_ADDR]] : i32
+    // CHECK: %[[VAL_LANE_ID:.+]] = spirv.ISub %[[LANE_ID]], %[[OFFSET]] : i32
+    // CHECK: %[[CST0:.+]] = spirv.Constant 0 : i32
+    // CHECK: %[[VALID:.+]] = spirv.SGreaterThanEqual %[[VAL_LANE_ID]], %[[CST0]] : i32
+
+    %result, %valid = gpu.shuffle up %val, %offset, %width : f32
+    gpu.return
+  }
+}
+
+}
-- 
cgit v1.2.3


From 2c2ad9a096e78e9129f8cb2d4ee260eb7e67473f Mon Sep 17 00:00:00 2001
From: Younan Zhang <zyn7109@gmail.com>
Date: Thu, 19 Jun 2025 14:59:32 +0800
Subject: Reapply "[Clang] Profile singly-resolved UnresolvedLookupExpr with
 the declaration" (#140680)

For a dependent variable template specialization, we don't build a
dependent Decl node or a DeclRefExpr to represent it. Instead, we
preserve the UnresolvedLookupExpr until instantiation.

However, this approach isn't ideal for constraint normalization. We
consider the qualifier during profiling, but since that's based on the
written code, it can introduce confusing differences, even when the
expressions resolve to the same declaration.

This change profiles the underlying VarTemplateDecl if
UnresolvedLookupExpr is used to model a dependent use of it.

Fixes https://github.com/llvm/llvm-project/issues/139476
---
 clang/docs/ReleaseNotes.rst                         |  1 +
 clang/lib/AST/StmtProfile.cpp                       | 10 ++++++++--
 clang/test/SemaCXX/exception-spec.cpp               | 21 +++++++++++++++++++++
 .../test/SemaTemplate/concepts-out-of-line-def.cpp  | 15 +++++++++++++++
 4 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 18234188101f..dd748ab06873 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -852,6 +852,7 @@ Bug Fixes to C++ Support
 - Fixed the handling of pack indexing types in the constraints of a member function redeclaration. (#GH138255)
 - Clang now correctly parses arbitrary order of ``[[]]``, ``__attribute__`` and ``alignas`` attributes for declarations (#GH133107)
 - Fixed a crash when forming an invalid function type in a dependent context. (#GH138657) (#GH115725) (#GH68852)
+- Fixed a function declaration mismatch that caused inconsistencies between concepts and variable template declarations. (#GH139476)
 - Clang no longer segfaults when there is a configuration mismatch between modules and their users (http://crbug.com/400353616).
 - Fix an incorrect deduction when calling an explicit object member function template through an overload set address.
 - Fixed bug in constant evaluation that would allow using the value of a
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index c666d966a6e5..c61450e19f1b 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -2189,8 +2189,14 @@ StmtProfiler::VisitCXXPseudoDestructorExpr(const CXXPseudoDestructorExpr *S) {
 
 void StmtProfiler::VisitOverloadExpr(const OverloadExpr *S) {
   VisitExpr(S);
-  VisitNestedNameSpecifier(S->getQualifier());
-  VisitName(S->getName(), /*TreatAsDecl*/ true);
+  bool DescribingDependentVarTemplate =
+      S->getNumDecls() == 1 && isa<VarTemplateDecl>(*S->decls_begin());
+  if (DescribingDependentVarTemplate) {
+    VisitDecl(*S->decls_begin());
+  } else {
+    VisitNestedNameSpecifier(S->getQualifier());
+    VisitName(S->getName(), /*TreatAsDecl*/ true);
+  }
   ID.AddBoolean(S->hasExplicitTemplateArgs());
   if (S->hasExplicitTemplateArgs())
     VisitTemplateArguments(S->getTemplateArgs(), S->getNumTemplateArgs());
diff --git a/clang/test/SemaCXX/exception-spec.cpp b/clang/test/SemaCXX/exception-spec.cpp
index 6ad19aab397b..31c691b28da4 100644
--- a/clang/test/SemaCXX/exception-spec.cpp
+++ b/clang/test/SemaCXX/exception-spec.cpp
@@ -52,3 +52,24 @@ namespace AssignmentOp {
     D2 &operator=(const D2&); // expected-error {{more lax}}
   };
 }
+
+namespace OverloadedFunctions {
+
+template <typename T>
+void f(T&) noexcept;
+
+template <typename T, int N>
+void f(T (&arr)[N]) noexcept(noexcept(f(*arr)));
+
+template <typename T>
+inline void f(T&) noexcept {}
+
+template <typename T, int N>
+inline void f(T (&arr)[N]) noexcept(noexcept(f(*arr))) {}
+
+void g() {
+    int x[1];
+    f(x);
+}
+
+}
diff --git a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
index e5d00491d3fb..bf505dec0ca1 100644
--- a/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
+++ b/clang/test/SemaTemplate/concepts-out-of-line-def.cpp
@@ -853,3 +853,18 @@ template <int... Ts>
 requires C<Ts...[0]>
 auto TplClass<int>::buggy() -> void {}
 }
+
+namespace GH139476 {
+
+namespace moo {
+  template <typename T>
+  constexpr bool baa = true;
+
+  template <typename T> requires baa<T>
+  void caw();
+}
+
+template <typename T> requires moo::baa<T>
+void moo::caw() {}
+
+}
-- 
cgit v1.2.3


From 9ee55e717308757b580dff182fc23b40d1c18a56 Mon Sep 17 00:00:00 2001
From: Amr Hesham <amr96@programmer.net>
Date: Thu, 19 Jun 2025 09:01:51 +0200
Subject: [CIR] Implement folder for VecSplatOp (#143771)

This change adds a folder for the VecSplatOp

Issue https://github.com/llvm/llvm-project/issues/136487
---
 clang/lib/CIR/Dialect/Transforms/CIRSimplify.cpp   | 30 ++++++++++++++++++++--
 .../lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp  |  4 +--
 clang/test/CIR/Transforms/vector-splat.cir         | 16 ++++++++++++
 3 files changed, 45 insertions(+), 5 deletions(-)
 create mode 100644 clang/test/CIR/Transforms/vector-splat.cir

diff --git a/clang/lib/CIR/Dialect/Transforms/CIRSimplify.cpp b/clang/lib/CIR/Dialect/Transforms/CIRSimplify.cpp
index 67ed4124f26c..3b7f08c44140 100644
--- a/clang/lib/CIR/Dialect/Transforms/CIRSimplify.cpp
+++ b/clang/lib/CIR/Dialect/Transforms/CIRSimplify.cpp
@@ -260,6 +260,31 @@ struct SimplifySwitch : public OpRewritePattern<SwitchOp> {
   }
 };
 
+struct SimplifyVecSplat : public OpRewritePattern<VecSplatOp> {
+  using OpRewritePattern<VecSplatOp>::OpRewritePattern;
+  LogicalResult matchAndRewrite(VecSplatOp op,
+                                PatternRewriter &rewriter) const override {
+    mlir::Value splatValue = op.getValue();
+    auto constant =
+        mlir::dyn_cast_if_present<cir::ConstantOp>(splatValue.getDefiningOp());
+    if (!constant)
+      return mlir::failure();
+
+    auto value = constant.getValue();
+    if (!mlir::isa_and_nonnull<cir::IntAttr>(value) &&
+        !mlir::isa_and_nonnull<cir::FPAttr>(value))
+      return mlir::failure();
+
+    cir::VectorType resultType = op.getResult().getType();
+    SmallVector<mlir::Attribute, 16> elements(resultType.getSize(), value);
+    auto constVecAttr = cir::ConstVectorAttr::get(
+        resultType, mlir::ArrayAttr::get(getContext(), elements));
+
+    rewriter.replaceOpWithNewOp<cir::ConstantOp>(op, constVecAttr);
+    return mlir::success();
+  }
+};
+
 //===----------------------------------------------------------------------===//
 // CIRSimplifyPass
 //===----------------------------------------------------------------------===//
@@ -275,7 +300,8 @@ void populateMergeCleanupPatterns(RewritePatternSet &patterns) {
   patterns.add<
     SimplifyTernary,
     SimplifySelect,
-    SimplifySwitch
+    SimplifySwitch,
+    SimplifyVecSplat
   >(patterns.getContext());
   // clang-format on
 }
@@ -288,7 +314,7 @@ void CIRSimplifyPass::runOnOperation() {
   // Collect operations to apply patterns.
   llvm::SmallVector<Operation *, 16> ops;
   getOperation()->walk([&](Operation *op) {
-    if (isa<TernaryOp, SelectOp, SwitchOp>(op))
+    if (isa<TernaryOp, SelectOp, SwitchOp, VecSplatOp>(op))
       ops.push_back(op);
   });
 
diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
index b73cb839828e..5f41e340e247 100644
--- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
+++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
@@ -979,9 +979,7 @@ mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
     }
 
     attr = rewriter.getArrayAttr(components);
-  }
-
-  else {
+  } else {
     return op.emitError() << "unsupported constant type " << op.getType();
   }
 
diff --git a/clang/test/CIR/Transforms/vector-splat.cir b/clang/test/CIR/Transforms/vector-splat.cir
new file mode 100644
index 000000000000..e2274b8627b1
--- /dev/null
+++ b/clang/test/CIR/Transforms/vector-splat.cir
@@ -0,0 +1,16 @@
+// RUN: cir-opt %s -cir-simplify -o - | FileCheck %s
+
+!s32i = !cir.int<s, 32>
+
+module  {
+  cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+    %v = cir.const #cir.int<3> : !s32i
+    %vec = cir.vec.splat %v : !s32i, !cir.vector<4 x !s32i>
+    cir.return %vec : !cir.vector<4 x !s32i>
+  }
+
+  // CHECK: cir.func @fold_shuffle_vector_op_test() -> !cir.vector<4 x !s32i> {
+  // CHECK-NEXT: %0 = cir.const #cir.const_vector<[#cir.int<3> : !s32i, #cir.int<3> : !s32i,
+  // CHECK-SAME: #cir.int<3> : !s32i, #cir.int<3> : !s32i]> : !cir.vector<4 x !s32i>
+  // CHECK-NEXT: cir.return %0 : !cir.vector<4 x !s32i>
+}
-- 
cgit v1.2.3


From 408e55098d7d8f7064d7a288b5e3fe6fdbbc2ad4 Mon Sep 17 00:00:00 2001
From: quic_hchandel <quic_hchandel@quicinc.com>
Date: Thu, 19 Jun 2025 12:36:20 +0530
Subject: [RISCV] Add support for handling one tied operand in the source
 instruction for compress patterns (#143660)

This update enables compress patterns to handle one tied operand in
source instructions, which was previously unsupported. Qualcomm's uC
extension Xqci includes several instructions with tied operands that can
be compressed into smaller forms. This change adds the necessary support
to enable such compression. Additionally, a compress pattern for the
qc.muliadd instruction has been implemented.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |  5 +++
 llvm/test/MC/RISCV/xqciac-valid.s           | 21 ++++++++---
 llvm/utils/TableGen/CompressInstEmitter.cpp | 58 +++++++++++++++++++++--------
 3 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 09852c6fd596..2856df47f704 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -1570,6 +1570,11 @@ def : CompressPat<(QC_E_ADDI X2, X2, simm10_lsb0000nonzero:$imm),
                   (C_ADDI16SP X2, simm10_lsb0000nonzero:$imm)>;
 } // let isCompressOnly = true, Predicates = [HasVendorXqcilia, IsRV32]
 
+let Predicates = [HasVendorXqciac, IsRV32] in {
+def : CompressPat<(QC_MULIADD GPRC:$rd, GPRC:$rs1, uimm5:$imm5),
+                  (QC_C_MULIADD GPRC:$rd, GPRC:$rs1, uimm5:$imm5)>;
+}
+
 let isCompressOnly = true, Predicates = [HasVendorXqcibi, IsRV32] in {
 def : CompressPat<(QC_E_BEQI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12),
                   (QC_BEQI GPRNoX0:$rs1, simm5nonzero:$imm5, bare_simm13_lsb0:$imm12)>;
diff --git a/llvm/test/MC/RISCV/xqciac-valid.s b/llvm/test/MC/RISCV/xqciac-valid.s
index 438c4cafe0df..1afebc75cb45 100644
--- a/llvm/test/MC/RISCV/xqciac-valid.s
+++ b/llvm/test/MC/RISCV/xqciac-valid.s
@@ -1,24 +1,27 @@
 # Xqciac - Qualcomm uC Load-Store Address Calculation Extension
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqciac -M no-aliases -show-encoding \
-# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST,CHECK-NOALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqciac < %s \
 # RUN:     | llvm-objdump --mattr=+experimental-xqciac -M no-aliases --no-print-imm-hex -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 # RUN: llvm-mc %s -triple=riscv32 -mattr=+experimental-xqciac -show-encoding \
-# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST %s
+# RUN:     | FileCheck -check-prefixes=CHECK-ENC,CHECK-INST,CHECK-ALIAS %s
 # RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+experimental-xqciac < %s \
 # RUN:     | llvm-objdump --mattr=+experimental-xqciac --no-print-imm-hex -d - \
 # RUN:     | FileCheck -check-prefix=CHECK-INST %s
 
-# CHECK-INST: qc.c.muliadd    a0, a1, 0
+# CHECK-NOALIAS: qc.c.muliadd    a0, a1, 0
+# CHECK-ALIAS: qc.muliadd    a0, a1, 0
 # CHECK-ENC: encoding: [0x8a,0x21]
 qc.c.muliadd x10, x11, 0
 
-# CHECK-INST: qc.c.muliadd    a0, a1, 31
+# CHECK-NOALIAS: qc.c.muliadd    a0, a1, 31
+# CHECK-ALIAS: qc.muliadd    a0, a1, 31
 # CHECK-ENC: encoding: [0xea,0x3d]
 qc.c.muliadd x10, x11, 31
 
-# CHECK-INST: qc.c.muliadd    a0, a1, 16
+# CHECK-NOALIAS: qc.c.muliadd    a0, a1, 16
+# CHECK-ALIAS: qc.muliadd    a0, a1, 16
 # CHECK-ENC: encoding: [0xaa,0x21]
 qc.c.muliadd x10, x11, 16
 
@@ -47,3 +50,11 @@ qc.shladd x10, x11, x12, 4
 # CHECK-INST: qc.shladd       a0, a1, a2, 31
 # CHECK-ENC: encoding: [0x0b,0xb5,0xc5,0x7e]
 qc.shladd x10, x11, x12, 31
+
+# Check that compress pattern for qc.muliadd works
+
+# CHECK-NOALIAS: qc.c.muliadd    a0, a1, 16
+# CHECK-ALIAS: qc.muliadd    a0, a1, 16
+# CHECK-ENC: encoding: [0xaa,0x21]
+qc.muliadd x10, x11, 16
+
diff --git a/llvm/utils/TableGen/CompressInstEmitter.cpp b/llvm/utils/TableGen/CompressInstEmitter.cpp
index 4a0b6d79c53d..2dfeea36e213 100644
--- a/llvm/utils/TableGen/CompressInstEmitter.cpp
+++ b/llvm/utils/TableGen/CompressInstEmitter.cpp
@@ -75,6 +75,7 @@
 #include "llvm/TableGen/Error.h"
 #include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
+#include <limits>
 #include <set>
 #include <vector>
 using namespace llvm;
@@ -123,10 +124,10 @@ class CompressInstEmitter {
   const RecordKeeper &Records;
   const CodeGenTarget Target;
   std::vector<CompressPat> CompressPatterns;
-
   void addDagOperandMapping(const Record *Rec, const DagInit *Dag,
                             const CodeGenInstruction &Inst,
-                            IndexedMap<OpData> &OperandMap, bool IsSourceInst);
+                            IndexedMap<OpData> &OperandMap, bool IsSourceInst,
+                            unsigned *SourceLastTiedOpPtr);
   void evaluateCompressPat(const Record *Compress);
   void emitCompressInstEmitter(raw_ostream &OS, EmitterType EType);
   bool validateTypes(const Record *DagOpType, const Record *InstOpType,
@@ -143,7 +144,8 @@ class CompressInstEmitter {
                                 IndexedMap<OpData> &SourceOperandMap,
                                 IndexedMap<OpData> &DestOperandMap,
                                 StringMap<unsigned> &SourceOperands,
-                                const CodeGenInstruction &DestInst);
+                                const CodeGenInstruction &DestInst,
+                                unsigned SourceLastTiedOp);
 
 public:
   CompressInstEmitter(const RecordKeeper &R) : Records(R), Target(R) {}
@@ -206,7 +208,8 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
                                                const DagInit *Dag,
                                                const CodeGenInstruction &Inst,
                                                IndexedMap<OpData> &OperandMap,
-                                               bool IsSourceInst) {
+                                               bool IsSourceInst,
+                                               unsigned *SourceLastTiedOpPtr) {
   unsigned NumMIOperands = 0;
   for (const auto &Op : Inst.Operands)
     NumMIOperands += Op.MINumOperands;
@@ -219,12 +222,16 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
   // are represented.
   unsigned TiedCount = 0;
   unsigned OpNo = 0;
+  if (IsSourceInst)
+    *SourceLastTiedOpPtr = std::numeric_limits<unsigned int>::max();
   for (const auto &Opnd : Inst.Operands) {
     int TiedOpIdx = Opnd.getTiedRegister();
     if (-1 != TiedOpIdx) {
       // Set the entry in OperandMap for the tied operand we're skipping.
       OperandMap[OpNo].Kind = OperandMap[TiedOpIdx].Kind;
       OperandMap[OpNo].Data = OperandMap[TiedOpIdx].Data;
+      if (IsSourceInst)
+        *SourceLastTiedOpPtr = OpNo;
       ++OpNo;
       ++TiedCount;
       continue;
@@ -289,15 +296,23 @@ void CompressInstEmitter::addDagOperandMapping(const Record *Rec,
 static bool verifyDagOpCount(const CodeGenInstruction &Inst, const DagInit *Dag,
                              bool IsSource) {
   unsigned NumMIOperands = 0;
-  for (const auto &Op : Inst.Operands)
+
+  // Use this to count number of tied Operands in Source Inst in this function.
+  // This counter is required here to error out when there is a Source
+  // Inst with two or more tied operands.
+  unsigned SourceInstTiedOpCount = 0;
+  for (const auto &Op : Inst.Operands) {
     NumMIOperands += Op.MINumOperands;
+    if (Op.getTiedRegister() != -1)
+      SourceInstTiedOpCount++;
+  }
 
   if (Dag->getNumArgs() == NumMIOperands)
     return true;
 
-  // Source instructions are non compressed instructions and don't have tied
-  // operands.
-  if (IsSource)
+  // Source instructions are non compressed instructions and have at most one
+  // tied operand.
+  if (IsSource && (SourceInstTiedOpCount >= 2))
     PrintFatalError(Inst.TheDef->getLoc(),
                     "Input operands for Inst '" + Inst.TheDef->getName() +
                         "' and input Dag operand count mismatch");
@@ -381,7 +396,8 @@ void CompressInstEmitter::createDagOperandMapping(
 void CompressInstEmitter::createInstOperandMapping(
     const Record *Rec, const DagInit *SourceDag, const DagInit *DestDag,
     IndexedMap<OpData> &SourceOperandMap, IndexedMap<OpData> &DestOperandMap,
-    StringMap<unsigned> &SourceOperands, const CodeGenInstruction &DestInst) {
+    StringMap<unsigned> &SourceOperands, const CodeGenInstruction &DestInst,
+    unsigned SourceLastTiedOp) {
   // TiedCount keeps track of the number of operands skipped in Inst
   // operands list to get to the corresponding Dag operand.
   unsigned TiedCount = 0;
@@ -422,10 +438,18 @@ void CompressInstEmitter::createInstOperandMapping(
       assert(DestDag->getArgNameStr(DagArgIdx) ==
                  SourceDag->getArgNameStr(SourceOp->getValue()) &&
              "Incorrect operand mapping detected!\n");
-      DestOperandMap[OpNo].Data.Operand = SourceOp->getValue();
-      SourceOperandMap[SourceOp->getValue()].Data.Operand = OpNo;
-      LLVM_DEBUG(dbgs() << "    " << SourceOp->getValue() << " ====> " << OpNo
-                        << "\n");
+
+      // Following four lines ensure the correct handling of a single tied
+      // operand in the Source Inst. SourceDagOp points to the position of
+      // appropriate Dag argument which is not correct in presence of tied
+      // operand in the Source Inst and must be incremented by 1 to reflect
+      // correct position of the operand in Source Inst
+      unsigned SourceDagOp = SourceOp->getValue();
+      if (SourceDagOp >= SourceLastTiedOp)
+        SourceDagOp++;
+      DestOperandMap[OpNo].Data.Operand = SourceDagOp;
+      SourceOperandMap[SourceDagOp].Data.Operand = OpNo;
+      LLVM_DEBUG(dbgs() << "    " << SourceDagOp << " ====> " << OpNo << "\n");
     }
   }
 }
@@ -484,15 +508,16 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
   // Fill the mapping from the source to destination instructions.
 
   IndexedMap<OpData> SourceOperandMap;
+  unsigned SourceLastTiedOp; // postion of the last tied operand in Source Inst
   // Create a mapping between source Dag operands and source Inst operands.
   addDagOperandMapping(Rec, SourceDag, SourceInst, SourceOperandMap,
-                       /*IsSourceInst*/ true);
+                       /*IsSourceInst*/ true, &SourceLastTiedOp);
 
   IndexedMap<OpData> DestOperandMap;
   // Create a mapping between destination Dag operands and destination Inst
   // operands.
   addDagOperandMapping(Rec, DestDag, DestInst, DestOperandMap,
-                       /*IsSourceInst*/ false);
+                       /*IsSourceInst*/ false, nullptr);
 
   StringMap<unsigned> SourceOperands;
   StringMap<unsigned> DestOperands;
@@ -500,7 +525,8 @@ void CompressInstEmitter::evaluateCompressPat(const Record *Rec) {
                           SourceOperandMap);
   // Create operand mapping between the source and destination instructions.
   createInstOperandMapping(Rec, SourceDag, DestDag, SourceOperandMap,
-                           DestOperandMap, SourceOperands, DestInst);
+                           DestOperandMap, SourceOperands, DestInst,
+                           SourceLastTiedOp);
 
   // Get the target features for the CompressPat.
   std::vector<const Record *> PatReqFeatures;
-- 
cgit v1.2.3


From 3e795c60c73e990fbbf254715cb47855c32bcfae Mon Sep 17 00:00:00 2001
From: Dmitry Vasilyev <dvassiliev@accesssoftek.com>
Date: Thu, 19 Jun 2025 11:12:34 +0400
Subject: [lldb] Disable TestTargetWatchAddress on Windows x86_64 (#144779)

See #144777 for details.
---
 .../watchpoint/watchlocation/TestTargetWatchAddress.py         | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
index 37fa911b3714..f1c7a60300df 100644
--- a/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
+++ b/lldb/test/API/python_api/watchpoint/watchlocation/TestTargetWatchAddress.py
@@ -21,6 +21,11 @@ class TargetWatchpointCreateByAddressPITestCase(TestBase):
         # This is for verifying that watch location works.
         self.violating_func = "do_bad_thing_with_location"
 
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/144777",
+    )
     def test_watch_create_by_address(self):
         """Exercise SBTarget.WatchpointCreateByAddress() API to set a watchpoint."""
         self.build()
@@ -88,6 +93,11 @@ class TargetWatchpointCreateByAddressPITestCase(TestBase):
 
         # This finishes our test.
 
+    @skipIf(
+        oslist=["windows"],
+        archs=["x86_64"],
+        bugnumber="github.com/llvm/llvm-project/issues/144777",
+    )
     def test_watch_address(self):
         """Exercise SBTarget.WatchAddress() API to set a watchpoint.
         Same as test_watch_create_by_address, but uses the simpler API.
-- 
cgit v1.2.3


From a9a71b6d311892d6add6aab3790b20fe945cca38 Mon Sep 17 00:00:00 2001
From: "S. B. Tam" <cpplearner@outlook.com>
Date: Thu, 19 Jun 2025 15:21:28 +0800
Subject: [libc++][test] Don't pass ill-formed UTF-8 to MAKE_STRING_VIEW
 (#136403)

---
 .../escaped_output.unicode.pass.cpp                |  2 +-
 .../format/format.functions/fill.unicode.pass.cpp  | 36 ++++++++++++++--------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp
index c4adf601c40a..eb27c7095466 100644
--- a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp
@@ -337,7 +337,7 @@ void test_string() {
 
   // Ill-formed
   if constexpr (sizeof(CharT) == 1)
-    test_format(SV(R"("\x{80}")"), SV("{:?}"), SV("\x80"));
+    test_format(SV(R"("\x{80}")"), SV("{:?}"), "\x80");
 
   // *** P2713R1 examples ***
   test_format(SV(R"(["\u{301}"])"), SV("[{:?}]"), SV("\u0301"));
diff --git a/libcxx/test/std/utilities/format/format.functions/fill.unicode.pass.cpp b/libcxx/test/std/utilities/format/format.functions/fill.unicode.pass.cpp
index cd555e1ab9ce..76f756ae9148 100644
--- a/libcxx/test/std/utilities/format/format.functions/fill.unicode.pass.cpp
+++ b/libcxx/test/std/utilities/format/format.functions/fill.unicode.pass.cpp
@@ -75,30 +75,40 @@ void test() {
 
   // Invalid Unicode Scalar Values
   if constexpr (std::same_as<CharT, char>) {
-    check_exception("The format specifier contains malformed Unicode characters", SV("{:\xed\xa0\x80^}"), 42); // U+D800
-    check_exception("The format specifier contains malformed Unicode characters", SV("{:\xed\xa0\xbf^}"), 42); // U+DBFF
-    check_exception("The format specifier contains malformed Unicode characters", SV("{:\xed\xbf\x80^}"), 42); // U+DC00
-    check_exception("The format specifier contains malformed Unicode characters", SV("{:\xed\xbf\xbf^}"), 42); // U+DFFF
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xed\xa0\x80^}"},
+                    42); // U+D800
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xed\xa0\xbf^}"},
+                    42); // U+DBFF
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xed\xbf\x80^}"},
+                    42); // U+DC00
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xed\xbf\xbf^}"},
+                    42); // U+DFFF
 
-    check_exception(
-        "The format specifier contains malformed Unicode characters", SV("{:\xf4\x90\x80\x80^}"), 42); // U+110000
-    check_exception(
-        "The format specifier contains malformed Unicode characters", SV("{:\xf4\x90\xbf\xbf^}"), 42); // U+11FFFF
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xf4\x90\x80\x80^}"},
+                    42); // U+110000
+    check_exception("The format specifier contains malformed Unicode characters",
+                    std::string_view{"{:\xf4\x90\xbf\xbf^}"},
+                    42); // U+11FFFF
 
     check_exception("The format specifier contains malformed Unicode characters",
-                    SV("{:\x80^}"),
+                    std::string_view{"{:\x80^}"},
                     42); // Trailing code unit with no leading one.
     check_exception("The format specifier contains malformed Unicode characters",
-                    SV("{:\xc0^}"),
+                    std::string_view{"{:\xc0^}"},
                     42); // Missing trailing code unit.
     check_exception("The format specifier contains malformed Unicode characters",
-                    SV("{:\xe0\x80^}"),
+                    std::string_view{"{:\xe0\x80^}"},
                     42); // Missing trailing code unit.
     check_exception("The format specifier contains malformed Unicode characters",
-                    SV("{:\xf0\x80^}"),
+                    std::string_view{"{:\xf0\x80^}"},
                     42); // Missing two trailing code units.
     check_exception("The format specifier contains malformed Unicode characters",
-                    SV("{:\xf0\x80\x80^}"),
+                    std::string_view{"{:\xf0\x80\x80^}"},
                     42); // Missing trailing code unit.
 
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
-- 
cgit v1.2.3


From 50a7511138a42d2c7a69b68237ce88cc027b91bc Mon Sep 17 00:00:00 2001
From: Paschalis Mpeis <paschalis.mpeis@arm.com>
Date: Thu, 19 Jun 2025 08:51:08 +0100
Subject: [BOLT][AArch64] Fix PREL Relocs on RHEL8 (#144505)

---
 bolt/test/AArch64/r_aarch64_prelxx.s | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bolt/test/AArch64/r_aarch64_prelxx.s b/bolt/test/AArch64/r_aarch64_prelxx.s
index 5cbe2c50b294..39f74301cedf 100644
--- a/bolt/test/AArch64/r_aarch64_prelxx.s
+++ b/bolt/test/AArch64/r_aarch64_prelxx.s
@@ -5,7 +5,7 @@
 // REQUIRES: system-linux
 
 // RUN: %clang %cflags -nostartfiles -nostdlib %s -o %t.exe -mlittle-endian \
-// RUN:     -Wl,-q -Wl,-z,max-page-size=4
+// RUN:     -Wl,-q -Wl,-z,max-page-size=4 -Wl,--no-relax
 // RUN: llvm-readelf -Wa %t.exe | FileCheck %s -check-prefix=CHECKPREL
 
 // CHECKPREL:       R_AARCH64_PREL16      {{.*}} .dummy + 0
@@ -36,9 +36,9 @@
   .type _start, %function
 _start:
   adrp x0, datatable
-  add x0, x0, :lo12:datable
+  add x0, x0, :lo12:datatable
   mov x0, #0
-  ret 
+  ret
 
 .section .dummy, "a", @progbits
 dummy:
-- 
cgit v1.2.3


From e73bff89ef8e3c8cdd8895cdc3d021fc4dcabd76 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 19 Jun 2025 09:56:07 +0200
Subject: [AMDGPU] New RegBankSelect: Handle all 32/64 bit pointer types for
 B32/B64 rule (#142560)

The previous system explicitly enumerated the types. P0 was missing and thus we couldn't handle a select of P0s for example.
Generalize the logic to simply check the width of the pointer for 32/64 bit pointers, this should handle all common address spaces
---
 .../Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp   |   8 +-
 .../AMDGPU/GlobalISel/regbankselect-select.mir     | 196 +++++++++++++++++++++
 2 files changed, 199 insertions(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 89056b0271f1..5e21f44f7d54 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -199,13 +199,11 @@ UniformityLLTOpPredicateID LLTToId(LLT Ty) {
 
 UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
   if (Ty == LLT::scalar(32) || Ty == LLT::fixed_vector(2, 16) ||
-      Ty == LLT::pointer(3, 32) || Ty == LLT::pointer(5, 32) ||
-      Ty == LLT::pointer(6, 32))
+      (Ty.isPointer() && Ty.getSizeInBits() == 32))
     return B32;
   if (Ty == LLT::scalar(64) || Ty == LLT::fixed_vector(2, 32) ||
-      Ty == LLT::fixed_vector(4, 16) || Ty == LLT::pointer(1, 64) ||
-      Ty == LLT::pointer(4, 64) ||
-      (Ty.isPointer() && Ty.getAddressSpace() > AMDGPUAS::MAX_AMDGPU_ADDRESS))
+      Ty == LLT::fixed_vector(4, 16) ||
+      (Ty.isPointer() && Ty.getSizeInBits() == 64))
     return B64;
   if (Ty == LLT::fixed_vector(3, 32))
     return B96;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
index 762f7b950036..2fd2e03471f1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-select.mir
@@ -896,6 +896,31 @@ body: |
     %5:_(<4 x s16>) = G_SELECT %4, %2, %3
 ...
 
+---
+name: select_p0_scc_ss
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
+    ; CHECK-LABEL: name: select_p0_scc_ss
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:sgpr(p0) = COPY $sgpr4_sgpr5
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 1
+    ; CHECK-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[ICMP]], [[C]]
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:sgpr(p0) = G_SELECT [[AND]](s32), [[COPY2]], [[COPY3]]
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(p0) = COPY $sgpr2_sgpr3
+    %3:_(p0) = COPY $sgpr4_sgpr5
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p0) = G_SELECT %4, %2, %3
+...
+
 ---
 name: select_p1_scc_ss
 legalized: true
@@ -946,6 +971,36 @@ body: |
     %5:_(p999) = G_SELECT %4, %2, %3
 ...
 
+---
+name: select_p0_scc_sv
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: select_p0_scc_sv
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY2]](p0)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(p0) = COPY $sgpr2_sgpr3
+    %3:_(p0) = COPY $vgpr0_vgpr1
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p0) = G_SELECT %4, %2, %3
+
+...
+
 ---
 name: select_p1_scc_sv
 legalized: true
@@ -976,6 +1031,35 @@ body: |
 
 ...
 
+---
+name: select_p0_scc_vs
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
+    ; CHECK-LABEL: name: select_p0_scc_vs
+    ; CHECK: liveins: $sgpr0, $sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(s32) = COPY $sgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:sgpr(s32) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[AMDGPU_COPY_VCC_SCC:%[0-9]+]]:vcc(s1) = G_AMDGPU_COPY_VCC_SCC [[ICMP]](s32)
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY2]](p0)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[AMDGPU_COPY_VCC_SCC]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(p0) = COPY $sgpr2_sgpr3
+    %3:_(p0) = COPY $vgpr0_vgpr1
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p0) = G_SELECT %4, %3, %2
+...
+
 ---
 name: select_p1_scc_vs
 legalized: true
@@ -1033,6 +1117,35 @@ body: |
     %5:_(p1) = G_SELECT %4, %2, %3
 ...
 
+---
+name: select_p0_vcc_ss
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
+    ; CHECK-LABEL: name: select_p0_vcc_ss
+    ; CHECK: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0, $vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:sgpr(p0) = COPY $sgpr2_sgpr3
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY2]](s32), [[COPY3]]
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK-NEXT: [[COPY5:%[0-9]+]]:vgpr(p0) = COPY [[COPY1]](p0)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY5]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(p0) = COPY $sgpr2_sgpr3
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = COPY $vgpr1
+    %4:_(s1) = G_ICMP intpred(ne), %2, %3
+    %5:_(p0) = G_SELECT %4, %0, %1
+...
+
 ---
 name: select_p1_vcc_ss
 legalized: true
@@ -1062,6 +1175,34 @@ body: |
     %5:_(p1) = G_SELECT %4, %0, %1
 ...
 
+---
+name: select_p0_vcc_sv
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: select_p0_vcc_sv
+    ; CHECK: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p0) = COPY $vgpr2_vgpr3
+    %4:_(s1) = G_ICMP intpred(ne), %1, %2
+    %5:_(p0) = G_SELECT %4, %0, %3
+...
+
 ---
 name: select_p1_vcc_sv
 legalized: true
@@ -1090,6 +1231,34 @@ body: |
     %5:_(p1) = G_SELECT %4, %0, %3
 ...
 
+---
+name: select_p0_vcc_vs
+legalized: true
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
+    ; CHECK-LABEL: name: select_p0_vcc_vs
+    ; CHECK: liveins: $sgpr0_sgpr1, $vgpr0, $vgpr1, $vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[COPY2]]
+    ; CHECK-NEXT: [[COPY4:%[0-9]+]]:vgpr(p0) = COPY [[COPY]](p0)
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY4]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s32) = COPY $vgpr0
+    %2:_(s32) = COPY $vgpr1
+    %3:_(p0) = COPY $vgpr2_vgpr3
+    %4:_(s1) = G_ICMP intpred(ne), %1, %2
+    %5:_(p0) = G_SELECT %4, %3, %0
+...
+
 ---
 name: select_p1_vcc_vs
 legalized: true
@@ -1118,6 +1287,33 @@ body: |
     %5:_(p1) = G_SELECT %4, %3, %0
 ...
 
+---
+name: select_p0_vcc_vv
+legalized: true
+body: |
+  bb.0:
+    liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; CHECK-LABEL: name: select_p0_vcc_vv
+    ; CHECK: liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vgpr4_vgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY $vgpr1
+    ; CHECK-NEXT: [[COPY2:%[0-9]+]]:vgpr(p0) = COPY $vgpr2_vgpr3
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(p0) = COPY $vgpr4_vgpr5
+    ; CHECK-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(ne), [[COPY]](s32), [[COPY1]]
+    ; CHECK-NEXT: [[UV:%[0-9]+]]:vgpr(s32), [[UV1:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY2]](p0)
+    ; CHECK-NEXT: [[UV2:%[0-9]+]]:vgpr(s32), [[UV3:%[0-9]+]]:vgpr(s32) = G_UNMERGE_VALUES [[COPY3]](p0)
+    ; CHECK-NEXT: [[SELECT:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV]], [[UV2]]
+    ; CHECK-NEXT: [[SELECT1:%[0-9]+]]:vgpr(s32) = G_SELECT [[ICMP]](s1), [[UV1]], [[UV3]]
+    ; CHECK-NEXT: [[MV:%[0-9]+]]:vgpr(p0) = G_MERGE_VALUES [[SELECT]](s32), [[SELECT1]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p0) = COPY $vgpr2_vgpr3
+    %3:_(p0) = COPY $vgpr4_vgpr5
+    %4:_(s1) = G_ICMP intpred(ne), %0, %1
+    %5:_(p0) = G_SELECT %4, %2, %3
+...
+
 ---
 name: select_p1_vcc_vv
 legalized: true
-- 
cgit v1.2.3


From db8e6fc64534e986f5bf96cceaa76cc5007ac1c7 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 19 Jun 2025 09:58:57 +0200
Subject: [AMDGPU] New RegBanKSelect: Add S128 types (#142601)

---
 llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp | 9 +++++++++
 llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp  | 6 ++++++
 llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h    | 5 +++++
 3 files changed, 20 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 7ff822c6f658..89af98263659 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -556,6 +556,9 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
   case Sgpr64:
   case Vgpr64:
     return LLT::scalar(64);
+  case Sgpr128:
+  case Vgpr128:
+    return LLT::scalar(128);
   case VgprP0:
     return LLT::pointer(0, 64);
   case SgprP1:
@@ -646,6 +649,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
   case Sgpr16:
   case Sgpr32:
   case Sgpr64:
+  case Sgpr128:
   case SgprP1:
   case SgprP3:
   case SgprP4:
@@ -678,6 +682,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
   case Vgpr16:
   case Vgpr32:
   case Vgpr64:
+  case Vgpr128:
   case VgprP0:
   case VgprP1:
   case VgprP3:
@@ -718,6 +723,7 @@ void RegBankLegalizeHelper::applyMappingDst(
     case Sgpr16:
     case Sgpr32:
     case Sgpr64:
+    case Sgpr128:
     case SgprP1:
     case SgprP3:
     case SgprP4:
@@ -728,6 +734,7 @@ void RegBankLegalizeHelper::applyMappingDst(
     case Vgpr16:
     case Vgpr32:
     case Vgpr64:
+    case Vgpr128:
     case VgprP0:
     case VgprP1:
     case VgprP3:
@@ -839,6 +846,7 @@ void RegBankLegalizeHelper::applyMappingSrc(
     case Sgpr16:
     case Sgpr32:
     case Sgpr64:
+    case Sgpr128:
     case SgprP1:
     case SgprP3:
     case SgprP4:
@@ -865,6 +873,7 @@ void RegBankLegalizeHelper::applyMappingSrc(
     case Vgpr16:
     case Vgpr32:
     case Vgpr64:
+    case Vgpr128:
     case VgprP0:
     case VgprP1:
     case VgprP3:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 5e21f44f7d54..672fc5b79abc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -50,6 +50,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32);
   case S64:
     return MRI.getType(Reg) == LLT::scalar(64);
+  case S128:
+    return MRI.getType(Reg) == LLT::scalar(128);
   case P0:
     return MRI.getType(Reg) == LLT::pointer(0, 64);
   case P1:
@@ -84,6 +86,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32) && MUI.isUniform(Reg);
   case UniS64:
     return MRI.getType(Reg) == LLT::scalar(64) && MUI.isUniform(Reg);
+  case UniS128:
+    return MRI.getType(Reg) == LLT::scalar(128) && MUI.isUniform(Reg);
   case UniP0:
     return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isUniform(Reg);
   case UniP1:
@@ -116,6 +120,8 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::scalar(32) && MUI.isDivergent(Reg);
   case DivS64:
     return MRI.getType(Reg) == LLT::scalar(64) && MUI.isDivergent(Reg);
+  case DivS128:
+    return MRI.getType(Reg) == LLT::scalar(128) && MUI.isDivergent(Reg);
   case DivP0:
     return MRI.getType(Reg) == LLT::pointer(0, 64) && MUI.isDivergent(Reg);
   case DivP1:
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index bddfb8dd1913..30b900d871f3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -39,16 +39,19 @@ enum UniformityLLTOpPredicateID {
   S16,
   S32,
   S64,
+  S128,
 
   UniS1,
   UniS16,
   UniS32,
   UniS64,
+  UniS128,
 
   DivS1,
   DivS16,
   DivS32,
   DivS64,
+  DivS128,
 
   // pointers
   P0,
@@ -117,6 +120,7 @@ enum RegBankLLTMappingApplyID {
   Sgpr16,
   Sgpr32,
   Sgpr64,
+  Sgpr128,
   SgprP1,
   SgprP3,
   SgprP4,
@@ -135,6 +139,7 @@ enum RegBankLLTMappingApplyID {
   Vgpr16,
   Vgpr32,
   Vgpr64,
+  Vgpr128,
   VgprP0,
   VgprP1,
   VgprP3,
-- 
cgit v1.2.3


From 26d4b3cb4ca2f882384d940f3dad28f8d79451eb Mon Sep 17 00:00:00 2001
From: Sudharsan Veeravalli <quic_svs@quicinc.com>
Date: Thu, 19 Jun 2025 13:29:57 +0530
Subject: [RISCV] Don't prefer QC_EXT for SEXT_INREG patterns when Zbb is
 enabled (#144837)

`Zbb` has the `sext.b` and `sext.h` instructions that are compressible.
---
 llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td |   8 +-
 llvm/test/CodeGen/RISCV/xqcibm-extract.ll   | 141 ++++++++++++++++++++++++++++
 2 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
index 2856df47f704..3d0cad7884fd 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXqci.td
@@ -1439,8 +1439,6 @@ def : SelectQCbi<SETUGE, uimm16nonzero, Select_GPRNoX0_Using_CC_UImm16NonZero_QC
 } // let Predicates = [HasVendorXqcibi, IsRV32], AddedComplexity = 2
 
 let Predicates = [HasVendorXqcibm, IsRV32] in {
-def : Pat<(sext_inreg (i32 GPR:$rs1), i16), (QC_EXT GPR:$rs1, 16, 0)>;
-def : Pat<(sext_inreg (i32 GPR:$rs1), i8), (QC_EXT GPR:$rs1, 8, 0)>;
 def : Pat<(sext_inreg (i32 GPR:$rs1), i1), (QC_EXT GPR:$rs1, 1, 0)>;
 
 // Prefer qc.extu to andi for the following cases since the former can be compressed
@@ -1452,6 +1450,12 @@ def : Pat<(i32 (and GPRNoX0:$rs, 1023)), (QC_EXTU GPRNoX0:$rs, 10, 0)>;
 def : Pat<(i32 (and GPRNoX0:$rs, 2047)), (QC_EXTU GPRNoX0:$rs, 11, 0)>;
 } // Predicates = [HasVendorXqcibm, IsRV32]
 
+// If Zbb is enabled sext.b/h is preferred since they are compressible
+let Predicates = [HasVendorXqcibm, NoStdExtZbb, IsRV32] in {
+def : Pat<(sext_inreg (i32 GPR:$rs1), i16), (QC_EXT GPR:$rs1, 16, 0)>;
+def : Pat<(sext_inreg (i32 GPR:$rs1), i8), (QC_EXT GPR:$rs1, 8, 0)>;
+} // Predicates = [HasVendorXqcibm, NoStdExtZbb, IsRV32]
+
 let Predicates = [HasVendorXqcibm, HasStdExtZbb, IsRV32] in {
 def: Pat<(i32 (cttz (not (i32 GPR:$rs1)))), (QC_CTO GPR:$rs1)>;
 def: Pat<(i32 (ctlz (not (i32 GPR:$rs1)))), (QC_CLO GPR:$rs1)>;
diff --git a/llvm/test/CodeGen/RISCV/xqcibm-extract.ll b/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
index edf6e9a2d501..481bfdd66643 100644
--- a/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
+++ b/llvm/test/CodeGen/RISCV/xqcibm-extract.ll
@@ -3,6 +3,8 @@
 ; RUN:   | FileCheck %s -check-prefixes=RV32I
 ; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm -verify-machineinstrs < %s \
 ; RUN:   | FileCheck %s -check-prefixes=RV32XQCIBM
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-xqcibm,+zbb -verify-machineinstrs < %s \
+; RUN:   | FileCheck %s -check-prefixes=RV32XQCIBMZBB
 
 define i32 @sexti1_i32(i1 %a) nounwind {
 ; RV32I-LABEL: sexti1_i32:
@@ -15,6 +17,11 @@ define i32 @sexti1_i32(i1 %a) nounwind {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti1_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 1, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %sext = sext i1 %a to i32
   ret i32 %sext
 }
@@ -30,6 +37,11 @@ define i32 @sexti1_i32_2(i32 %a) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti1_i32_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 1, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i32 %a, 31
   %shr = ashr exact i32 %shl, 31
   ret i32 %shr
@@ -47,6 +59,11 @@ define i32 @sexti8_i32(i8 %a) nounwind {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti8_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.b a0, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %sext = sext i8 %a to i32
   ret i32 %sext
 }
@@ -62,6 +79,11 @@ define i32 @sexti8_i32_2(i32 %a) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti8_i32_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.b a0, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i32 %a, 24
   %shr = ashr exact i32 %shl, 24
   ret i32 %shr
@@ -78,6 +100,11 @@ define i32 @sexti16_i32(i16 %a) nounwind {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 16, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti16_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.h a0, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %sext = sext i16 %a to i32
   ret i32 %sext
 }
@@ -93,6 +120,11 @@ define i32 @sexti16_i32_2(i32 %a) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 16, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti16_i32_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.h a0, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i32 %a, 16
   %shr = ashr exact i32 %shl, 16
   ret i32 %shr
@@ -111,6 +143,12 @@ define i64 @sexti1_i64(i64 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 0
 ; RV32XQCIBM-NEXT:    mv a1, a0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti1_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 1, 0
+; RV32XQCIBMZBB-NEXT:    mv a1, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i64 %a, 63
   %shr = ashr exact i64 %shl, 63
   ret i64 %shr
@@ -129,6 +167,12 @@ define i64 @sexti1_i64_2(i1 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 0
 ; RV32XQCIBM-NEXT:    mv a1, a0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti1_i64_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 1, 0
+; RV32XQCIBMZBB-NEXT:    mv a1, a0
+; RV32XQCIBMZBB-NEXT:    ret
   %1 = sext i1 %a to i64
   ret i64 %1
 }
@@ -146,6 +190,12 @@ define i64 @sexti8_i64(i64 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 0
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti8_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.b a0, a0
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i64 %a, 56
   %shr = ashr exact i64 %shl, 56
   ret i64 %shr
@@ -164,6 +214,12 @@ define i64 @sexti8_i64_2(i8 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 0
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti8_i64_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.b a0, a0
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %1 = sext i8 %a to i64
   ret i64 %1
 }
@@ -181,6 +237,12 @@ define i64 @sexti16_i64(i64 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 16, 0
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti16_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.h a0, a0
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i64 %a, 48
   %shr = ashr exact i64 %shl, 48
   ret i64 %shr
@@ -199,6 +261,12 @@ define i64 @sexti16_i64_2(i16 %a) {
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 16, 0
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti16_i64_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    sext.h a0, a0
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %1 = sext i16 %a to i64
   ret i64 %1
 }
@@ -213,6 +281,11 @@ define i64 @sexti32_i64(i64 %a) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti32_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i64 %a, 32
   %shr = ashr exact i64 %shl, 32
   ret i64 %shr
@@ -228,6 +301,11 @@ define i64 @sexti32_i64_2(i32 %a) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    srai a1, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: sexti32_i64_2:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    srai a1, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %1 = sext i32 %a to i64
   ret i64 %1
 }
@@ -243,6 +321,11 @@ define i32 @extu_from_and_i32(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 12, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %a = and i32 %x, 4095
   ret i32 %a
 }
@@ -257,6 +340,11 @@ define i32 @no_extu_from_and_i32(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    andi a0, a0, 31
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: no_extu_from_and_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    andi a0, a0, 31
+; RV32XQCIBMZBB-NEXT:    ret
   %a = and i32 %x, 31
   ret i32 %a
 }
@@ -271,6 +359,11 @@ define i32 @extu_from_and_i32_simm12_lb(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 6, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_i32_simm12_lb:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 6, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %a = and i32 %x, 63
   ret i32 %a
 }
@@ -285,6 +378,11 @@ define i32 @extu_from_and_i32_simm12_ub(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 11, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_i32_simm12_ub:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 11, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %a = and i32 %x, 2047
   ret i32 %a
 }
@@ -302,6 +400,12 @@ define i64 @extu_from_and_i64(i64 %x) {
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 0
 ; RV32XQCIBM-NEXT:    li a1, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 12, 0
+; RV32XQCIBMZBB-NEXT:    li a1, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %a = and i64 %x, 4095
   ret i64 %a
 }
@@ -317,6 +421,11 @@ define i32 @extu_from_and_lshr_i32(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 3, 23
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_lshr_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 3, 23
+; RV32XQCIBMZBB-NEXT:    ret
   %shifted = lshr i32 %x, 23
   %masked = and i32 %shifted, 7
   ret i32 %masked
@@ -335,6 +444,12 @@ define i64 @extu_from_and_lshr_i64(i64 %x) {
 ; RV32XQCIBM-NEXT:    qc.extu a0, a1, 12, 14
 ; RV32XQCIBM-NEXT:    li a1, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_and_lshr_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a1, 12, 14
+; RV32XQCIBMZBB-NEXT:    li a1, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %shifted = lshr i64 %x, 46
   %masked = and i64 %shifted, 4095
   ret i64 %masked
@@ -351,6 +466,11 @@ define i32 @extu_from_lshr_and_i32(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 12
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_lshr_and_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 12, 12
+; RV32XQCIBMZBB-NEXT:    ret
   %masked = and i32 %x, 16773120
   %shifted = lshr i32 %masked, 12
   ret i32 %shifted
@@ -369,6 +489,12 @@ define i64 @extu_from_lshr_and_i64(i64 %x) {
 ; RV32XQCIBM-NEXT:    qc.extu a0, a0, 12, 12
 ; RV32XQCIBM-NEXT:    li a1, 0
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: extu_from_lshr_and_i64:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.extu a0, a0, 12, 12
+; RV32XQCIBMZBB-NEXT:    li a1, 0
+; RV32XQCIBMZBB-NEXT:    ret
   %masked = and i64 %x, 16773120
   %shifted = lshr i64 %masked, 12
   ret i64 %shifted
@@ -385,6 +511,11 @@ define i32 @ext_from_ashr_shl_i32(i32 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 8, 16
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: ext_from_ashr_shl_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 8, 16
+; RV32XQCIBMZBB-NEXT:    ret
   %shl = shl i32 %x, 8
   %ashr = ashr i32 %shl, 24
   ret i32 %ashr
@@ -401,6 +532,11 @@ define i32 @ext_from_ashr_sexti8_i32(i8 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 3, 5
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: ext_from_ashr_sexti8_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 3, 5
+; RV32XQCIBMZBB-NEXT:    ret
   %sext = sext i8 %x to i32
   %ashr = ashr i32 %sext, 5
   ret i32 %ashr
@@ -417,6 +553,11 @@ define i32 @ext_from_ashr_sexti16_i32(i16 %x) {
 ; RV32XQCIBM:       # %bb.0:
 ; RV32XQCIBM-NEXT:    qc.ext a0, a0, 1, 15
 ; RV32XQCIBM-NEXT:    ret
+;
+; RV32XQCIBMZBB-LABEL: ext_from_ashr_sexti16_i32:
+; RV32XQCIBMZBB:       # %bb.0:
+; RV32XQCIBMZBB-NEXT:    qc.ext a0, a0, 1, 15
+; RV32XQCIBMZBB-NEXT:    ret
   %sext = sext i16 %x to i32
   %ashr = ashr i32 %sext, 24
   ret i32 %ashr
-- 
cgit v1.2.3


From 7ceea22a7adad5d21328839facbc6a6d0151e056 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 19 Jun 2025 10:06:38 +0200
Subject: [AMDGPU] New RegBankSelect: Add Ptr32/Ptr64/Ptr128 (#142602)

There's quite a few opcodes that do not care about the exact AS of the pointer, just its size.
Adding generic types for these will help reduce duplication in the rule definitions.

I also moved the usual B types to use the new `isAnyPtr` helper I added to make sure they're supersets of the `Ptr` cases
---
 .../Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp  | 42 +++++++++++++++++-----
 .../Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp   | 29 ++++++++++++---
 .../lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h | 19 ++++++++++
 3 files changed, 77 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
index 89af98263659..b2ddc6e88966 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp
@@ -595,17 +595,23 @@ LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
   case VgprB32:
   case UniInVgprB32:
     if (Ty == LLT::scalar(32) || Ty == LLT::fixed_vector(2, 16) ||
-        Ty == LLT::pointer(3, 32) || Ty == LLT::pointer(5, 32) ||
-        Ty == LLT::pointer(6, 32))
+        isAnyPtr(Ty, 32))
       return Ty;
     return LLT();
+  case SgprPtr32:
+  case VgprPtr32:
+    return isAnyPtr(Ty, 32) ? Ty : LLT();
+  case SgprPtr64:
+  case VgprPtr64:
+    return isAnyPtr(Ty, 64) ? Ty : LLT();
+  case SgprPtr128:
+  case VgprPtr128:
+    return isAnyPtr(Ty, 128) ? Ty : LLT();
   case SgprB64:
   case VgprB64:
   case UniInVgprB64:
     if (Ty == LLT::scalar(64) || Ty == LLT::fixed_vector(2, 32) ||
-        Ty == LLT::fixed_vector(4, 16) || Ty == LLT::pointer(0, 64) ||
-        Ty == LLT::pointer(1, 64) || Ty == LLT::pointer(4, 64) ||
-        (Ty.isPointer() && Ty.getAddressSpace() > AMDGPUAS::MAX_AMDGPU_ADDRESS))
+        Ty == LLT::fixed_vector(4, 16) || isAnyPtr(Ty, 64))
       return Ty;
     return LLT();
   case SgprB96:
@@ -619,7 +625,7 @@ LLT RegBankLegalizeHelper::getBTyFromID(RegBankLLTMappingApplyID ID, LLT Ty) {
   case VgprB128:
   case UniInVgprB128:
     if (Ty == LLT::scalar(128) || Ty == LLT::fixed_vector(4, 32) ||
-        Ty == LLT::fixed_vector(2, 64))
+        Ty == LLT::fixed_vector(2, 64) || isAnyPtr(Ty, 128))
       return Ty;
     return LLT();
   case SgprB256:
@@ -654,6 +660,9 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
   case SgprP3:
   case SgprP4:
   case SgprP5:
+  case SgprPtr32:
+  case SgprPtr64:
+  case SgprPtr128:
   case SgprV2S16:
   case SgprV2S32:
   case SgprV4S32:
@@ -688,6 +697,9 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
   case VgprP3:
   case VgprP4:
   case VgprP5:
+  case VgprPtr32:
+  case VgprPtr64:
+  case VgprPtr128:
   case VgprV2S16:
   case VgprV2S32:
   case VgprV4S32:
@@ -754,12 +766,18 @@ void RegBankLegalizeHelper::applyMappingDst(
     case SgprB128:
     case SgprB256:
     case SgprB512:
+    case SgprPtr32:
+    case SgprPtr64:
+    case SgprPtr128:
     case VgprB32:
     case VgprB64:
     case VgprB96:
     case VgprB128:
     case VgprB256:
-    case VgprB512: {
+    case VgprB512:
+    case VgprPtr32:
+    case VgprPtr64:
+    case VgprPtr128: {
       assert(Ty == getBTyFromID(MethodIDs[OpIdx], Ty));
       assert(RB == getRegBankFromID(MethodIDs[OpIdx]));
       break;
@@ -864,7 +882,10 @@ void RegBankLegalizeHelper::applyMappingSrc(
     case SgprB96:
     case SgprB128:
     case SgprB256:
-    case SgprB512: {
+    case SgprB512:
+    case SgprPtr32:
+    case SgprPtr64:
+    case SgprPtr128: {
       assert(Ty == getBTyFromID(MethodIDs[i], Ty));
       assert(RB == getRegBankFromID(MethodIDs[i]));
       break;
@@ -895,7 +916,10 @@ void RegBankLegalizeHelper::applyMappingSrc(
     case VgprB96:
     case VgprB128:
     case VgprB256:
-    case VgprB512: {
+    case VgprB512:
+    case VgprPtr32:
+    case VgprPtr64:
+    case VgprPtr128: {
       assert(Ty == getBTyFromID(MethodIDs[i], Ty));
       if (RB != VgprRB) {
         auto CopyToVgpr = B.buildCopy({VgprRB, Ty}, Reg);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 672fc5b79abc..5402129e4188 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -26,6 +26,10 @@
 using namespace llvm;
 using namespace AMDGPU;
 
+bool AMDGPU::isAnyPtr(LLT Ty, unsigned Width) {
+  return Ty.isPointer() && Ty.getSizeInBits() == Width;
+}
+
 RegBankLLTMapping::RegBankLLTMapping(
     std::initializer_list<RegBankLLTMappingApplyID> DstOpMappingList,
     std::initializer_list<RegBankLLTMappingApplyID> SrcOpMappingList,
@@ -62,6 +66,12 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::pointer(4, 64);
   case P5:
     return MRI.getType(Reg) == LLT::pointer(5, 32);
+  case Ptr32:
+    return isAnyPtr(MRI.getType(Reg), 32);
+  case Ptr64:
+    return isAnyPtr(MRI.getType(Reg), 64);
+  case Ptr128:
+    return isAnyPtr(MRI.getType(Reg), 128);
   case V2S32:
     return MRI.getType(Reg) == LLT::fixed_vector(2, 32);
   case V4S32:
@@ -98,6 +108,12 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isUniform(Reg);
   case UniP5:
     return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isUniform(Reg);
+  case UniPtr32:
+    return isAnyPtr(MRI.getType(Reg), 32) && MUI.isUniform(Reg);
+  case UniPtr64:
+    return isAnyPtr(MRI.getType(Reg), 64) && MUI.isUniform(Reg);
+  case UniPtr128:
+    return isAnyPtr(MRI.getType(Reg), 128) && MUI.isUniform(Reg);
   case UniV2S16:
     return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isUniform(Reg);
   case UniB32:
@@ -132,6 +148,12 @@ bool matchUniformityAndLLT(Register Reg, UniformityLLTOpPredicateID UniID,
     return MRI.getType(Reg) == LLT::pointer(4, 64) && MUI.isDivergent(Reg);
   case DivP5:
     return MRI.getType(Reg) == LLT::pointer(5, 32) && MUI.isDivergent(Reg);
+  case DivPtr32:
+    return isAnyPtr(MRI.getType(Reg), 32) && MUI.isDivergent(Reg);
+  case DivPtr64:
+    return isAnyPtr(MRI.getType(Reg), 64) && MUI.isDivergent(Reg);
+  case DivPtr128:
+    return isAnyPtr(MRI.getType(Reg), 128) && MUI.isDivergent(Reg);
   case DivV2S16:
     return MRI.getType(Reg) == LLT::fixed_vector(2, 16) && MUI.isDivergent(Reg);
   case DivB32:
@@ -205,15 +227,14 @@ UniformityLLTOpPredicateID LLTToId(LLT Ty) {
 
 UniformityLLTOpPredicateID LLTToBId(LLT Ty) {
   if (Ty == LLT::scalar(32) || Ty == LLT::fixed_vector(2, 16) ||
-      (Ty.isPointer() && Ty.getSizeInBits() == 32))
+      isAnyPtr(Ty, 32))
     return B32;
   if (Ty == LLT::scalar(64) || Ty == LLT::fixed_vector(2, 32) ||
-      Ty == LLT::fixed_vector(4, 16) ||
-      (Ty.isPointer() && Ty.getSizeInBits() == 64))
+      Ty == LLT::fixed_vector(4, 16) || isAnyPtr(Ty, 64))
     return B64;
   if (Ty == LLT::fixed_vector(3, 32))
     return B96;
-  if (Ty == LLT::fixed_vector(4, 32))
+  if (Ty == LLT::fixed_vector(4, 32) || isAnyPtr(Ty, 128))
     return B128;
   return _;
 }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
index 30b900d871f3..7243d75aa830 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h
@@ -15,6 +15,7 @@
 
 namespace llvm {
 
+class LLT;
 class MachineRegisterInfo;
 class MachineInstr;
 class GCNSubtarget;
@@ -26,6 +27,9 @@ using MachineUniformityInfo = GenericUniformityInfo<MachineSSAContext>;
 
 namespace AMDGPU {
 
+/// \returns true if \p Ty is a pointer type with size \p Width.
+bool isAnyPtr(LLT Ty, unsigned Width);
+
 // IDs used to build predicate for RegBankLegalizeRule. Predicate can have one
 // or more IDs and each represents a check for 'uniform or divergent' + LLT or
 // just LLT on register operand.
@@ -59,18 +63,27 @@ enum UniformityLLTOpPredicateID {
   P3,
   P4,
   P5,
+  Ptr32,
+  Ptr64,
+  Ptr128,
 
   UniP0,
   UniP1,
   UniP3,
   UniP4,
   UniP5,
+  UniPtr32,
+  UniPtr64,
+  UniPtr128,
 
   DivP0,
   DivP1,
   DivP3,
   DivP4,
   DivP5,
+  DivPtr32,
+  DivPtr64,
+  DivPtr128,
 
   // vectors
   V2S16,
@@ -125,6 +138,9 @@ enum RegBankLLTMappingApplyID {
   SgprP3,
   SgprP4,
   SgprP5,
+  SgprPtr32,
+  SgprPtr64,
+  SgprPtr128,
   SgprV2S16,
   SgprV4S32,
   SgprV2S32,
@@ -145,6 +161,9 @@ enum RegBankLLTMappingApplyID {
   VgprP3,
   VgprP4,
   VgprP5,
+  VgprPtr32,
+  VgprPtr64,
+  VgprPtr128,
   VgprV2S16,
   VgprV2S32,
   VgprB32,
-- 
cgit v1.2.3


From 52ff58c3300338876ae63126ce0d33331000f1ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20Manuel=20Martinez=20Caama=C3=B1o?=
 <jmartinezcaamao@gmail.com>
Date: Thu, 19 Jun 2025 10:06:50 +0200
Subject: Revert "[CUDA][HIP] Add a __device__ version of
 std::__glibcxx_assert_fail()" (#144850)

Reverts llvm/llvm-project#136133
---
 clang/lib/Headers/CMakeLists.txt                 |  1 -
 clang/lib/Headers/cuda_wrappers/bits/c++config.h | 51 ------------------------
 2 files changed, 52 deletions(-)
 delete mode 100644 clang/lib/Headers/cuda_wrappers/bits/c++config.h

diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index c96d209c1fc0..c1c9d2e8c7b7 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -341,7 +341,6 @@ set(cuda_wrapper_files
 )
 
 set(cuda_wrapper_bits_files
-  cuda_wrappers/bits/c++config.h
   cuda_wrappers/bits/shared_ptr_base.h
   cuda_wrappers/bits/basic_string.h
   cuda_wrappers/bits/basic_string.tcc
diff --git a/clang/lib/Headers/cuda_wrappers/bits/c++config.h b/clang/lib/Headers/cuda_wrappers/bits/c++config.h
deleted file mode 100644
index eafa13a9cc64..000000000000
--- a/clang/lib/Headers/cuda_wrappers/bits/c++config.h
+++ /dev/null
@@ -1,51 +0,0 @@
-// libstdc++ uses the non-constexpr function std::__glibcxx_assert_fail()
-// to trigger compilation errors when the __glibcxx_assert(cond) macro
-// is used in a constexpr context.
-// Compilation fails when using code from the libstdc++ (such as std::array) on
-// device code, since these assertions invoke a non-constexpr host function from
-// device code.
-//
-// To work around this issue, we declare our own device version of the function
-
-#ifndef __CLANG_CUDA_WRAPPERS_BITS_CPP_CONFIG
-#define __CLANG_CUDA_WRAPPERS_BITS_CPP_CONFIG
-
-#include_next <bits/c++config.h>
-
-#ifdef _LIBCPP_BEGIN_NAMESPACE_STD
-_LIBCPP_BEGIN_NAMESPACE_STD
-#else
-namespace std {
-#ifdef _GLIBCXX_BEGIN_NAMESPACE_VERSION
-_GLIBCXX_BEGIN_NAMESPACE_VERSION
-#endif
-
-#ifdef _GLIBCXX_VERBOSE_ASSERT
-__attribute__((device, noreturn)) inline void
-__glibcxx_assert_fail(const char *file, int line, const char *function,
-                      const char *condition) noexcept {
-  if (file && function && condition)
-    __builtin_printf("%s:%d: %s: Assertion '%s' failed.\n", file, line,
-                     function, condition);
-  else if (function)
-    __builtin_printf("%s: Undefined behavior detected.\n", function);
-  __builtin_abort();
-}
-#endif
-
-#endif
-__attribute__((device, noreturn, __always_inline__,
-               __visibility__("default"))) inline void
-__glibcxx_assert_fail(...) noexcept {
-  __builtin_abort();
-}
-#ifdef _LIBCPP_END_NAMESPACE_STD
-_LIBCPP_END_NAMESPACE_STD
-#else
-#ifdef _GLIBCXX_BEGIN_NAMESPACE_VERSION
-_GLIBCXX_END_NAMESPACE_VERSION
-#endif
-} // namespace std
-#endif
-
-#endif
-- 
cgit v1.2.3


From 650b451d0065c8ea6a1f87e7fdc6d07648729549 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 19 Jun 2025 10:06:59 +0200
Subject: [libc++] Simplify the implementation of pointer_traits a bit
 (#142260)

---
 libcxx/include/__memory/pointer_traits.h | 116 +++++++++----------------------
 1 file changed, 33 insertions(+), 83 deletions(-)

diff --git a/libcxx/include/__memory/pointer_traits.h b/libcxx/include/__memory/pointer_traits.h
index 879b387b9ad1..8c7f8dff1b76 100644
--- a/libcxx/include/__memory/pointer_traits.h
+++ b/libcxx/include/__memory/pointer_traits.h
@@ -16,11 +16,13 @@
 #include <__type_traits/conditional.h>
 #include <__type_traits/conjunction.h>
 #include <__type_traits/decay.h>
+#include <__type_traits/detected_or.h>
 #include <__type_traits/enable_if.h>
 #include <__type_traits/integral_constant.h>
 #include <__type_traits/is_class.h>
 #include <__type_traits/is_function.h>
 #include <__type_traits/is_void.h>
+#include <__type_traits/nat.h>
 #include <__type_traits/void_t.h>
 #include <__utility/declval.h>
 #include <__utility/forward.h>
@@ -34,67 +36,37 @@ _LIBCPP_PUSH_MACROS
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-// clang-format off
-#define _LIBCPP_CLASS_TRAITS_HAS_XXX(NAME, PROPERTY)                                                                   \
-  template <class _Tp, class = void>                                                                                   \
-  struct NAME : false_type {};                                                                                         \
-  template <class _Tp>                                                                                                 \
-  struct NAME<_Tp, __void_t<typename _Tp::PROPERTY> > : true_type {}
-// clang-format on
-
-_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_pointer, pointer);
-_LIBCPP_CLASS_TRAITS_HAS_XXX(__has_element_type, element_type);
-
-template <class _Ptr, bool = __has_element_type<_Ptr>::value>
-struct __pointer_traits_element_type {};
-
 template <class _Ptr>
-struct __pointer_traits_element_type<_Ptr, true> {
-  using type _LIBCPP_NODEBUG = typename _Ptr::element_type;
-};
+struct __pointer_traits_element_type_impl {};
 
 template <template <class, class...> class _Sp, class _Tp, class... _Args>
-struct __pointer_traits_element_type<_Sp<_Tp, _Args...>, true> {
-  using type _LIBCPP_NODEBUG = typename _Sp<_Tp, _Args...>::element_type;
-};
-
-template <template <class, class...> class _Sp, class _Tp, class... _Args>
-struct __pointer_traits_element_type<_Sp<_Tp, _Args...>, false> {
+struct __pointer_traits_element_type_impl<_Sp<_Tp, _Args...> > {
   using type _LIBCPP_NODEBUG = _Tp;
 };
 
-template <class _Tp, class = void>
-struct __has_difference_type : false_type {};
-
-template <class _Tp>
-struct __has_difference_type<_Tp, __void_t<typename _Tp::difference_type> > : true_type {};
-
-template <class _Ptr, bool = __has_difference_type<_Ptr>::value>
-struct __pointer_traits_difference_type {
-  using type _LIBCPP_NODEBUG = ptrdiff_t;
-};
+template <class _Ptr, class = void>
+struct __pointer_traits_element_type : __pointer_traits_element_type_impl<_Ptr> {};
 
 template <class _Ptr>
-struct __pointer_traits_difference_type<_Ptr, true> {
-  using type _LIBCPP_NODEBUG = typename _Ptr::difference_type;
+struct __pointer_traits_element_type<_Ptr, __void_t<typename _Ptr::element_type> > {
+  using type _LIBCPP_NODEBUG = typename _Ptr::element_type;
 };
 
 template <class _Tp, class _Up>
-struct __has_rebind {
-private:
-  template <class _Xp>
-  static false_type __test(...);
-  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
-  template <class _Xp>
-  static true_type __test(typename _Xp::template rebind<_Up>* = 0);
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
+struct __pointer_traits_rebind_impl {
+  static_assert(false, "Cannot rebind pointer; did you forget to add a rebind member to your pointer?");
+};
 
-public:
-  static const bool value = decltype(__test<_Tp>(0))::value;
+template <template <class, class...> class _Sp, class _Tp, class... _Args, class _Up>
+struct __pointer_traits_rebind_impl<_Sp<_Tp, _Args...>, _Up> {
+  using type _LIBCPP_NODEBUG = _Sp<_Up, _Args...>;
 };
 
-template <class _Tp, class _Up, bool = __has_rebind<_Tp, _Up>::value>
-struct __pointer_traits_rebind {
+template <class _Tp, class _Up, class = void>
+struct __pointer_traits_rebind : __pointer_traits_rebind_impl<_Tp, _Up> {};
+
+template <class _Tp, class _Up>
+struct __pointer_traits_rebind<_Tp, _Up, __void_t<typename _Tp::template rebind<_Up> > > {
 #ifndef _LIBCPP_CXX03_LANG
   using type _LIBCPP_NODEBUG = typename _Tp::template rebind<_Up>;
 #else
@@ -102,19 +74,8 @@ struct __pointer_traits_rebind {
 #endif
 };
 
-template <template <class, class...> class _Sp, class _Tp, class... _Args, class _Up>
-struct __pointer_traits_rebind<_Sp<_Tp, _Args...>, _Up, true> {
-#ifndef _LIBCPP_CXX03_LANG
-  using type _LIBCPP_NODEBUG = typename _Sp<_Tp, _Args...>::template rebind<_Up>;
-#else
-  using type _LIBCPP_NODEBUG = typename _Sp<_Tp, _Args...>::template rebind<_Up>::other;
-#endif
-};
-
-template <template <class, class...> class _Sp, class _Tp, class... _Args, class _Up>
-struct __pointer_traits_rebind<_Sp<_Tp, _Args...>, _Up, false> {
-  typedef _Sp<_Up, _Args...> type;
-};
+template <class _Tp>
+using __difference_type_member _LIBCPP_NODEBUG = typename _Tp::difference_type;
 
 template <class _Ptr, class = void>
 struct __pointer_traits_impl {};
@@ -123,7 +84,7 @@ template <class _Ptr>
 struct __pointer_traits_impl<_Ptr, __void_t<typename __pointer_traits_element_type<_Ptr>::type> > {
   typedef _Ptr pointer;
   typedef typename __pointer_traits_element_type<pointer>::type element_type;
-  typedef typename __pointer_traits_difference_type<pointer>::type difference_type;
+  using difference_type = __detected_or_t<ptrdiff_t, __difference_type_member, pointer>;
 
 #ifndef _LIBCPP_CXX03_LANG
   template <class _Up>
@@ -135,9 +96,6 @@ struct __pointer_traits_impl<_Ptr, __void_t<typename __pointer_traits_element_ty
   };
 #endif // _LIBCPP_CXX03_LANG
 
-private:
-  struct __nat {};
-
 public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
   pointer_to(__conditional_t<is_void<element_type>::value, __nat, element_type>& __r) {
@@ -164,9 +122,6 @@ struct pointer_traits<_Tp*> {
   };
 #endif
 
-private:
-  struct __nat {};
-
 public:
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static pointer
   pointer_to(__conditional_t<is_void<element_type>::value, __nat, element_type>& __r) _NOEXCEPT {
@@ -257,20 +212,26 @@ template <class _Tp>
 struct __pointer_of {};
 
 template <class _Tp>
-  requires(__has_pointer<_Tp>::value)
+concept __has_pointer_member = requires { typename _Tp::pointer; };
+
+template <class _Tp>
+concept __has_element_type_member = requires { typename _Tp::element_type; };
+
+template <class _Tp>
+  requires __has_pointer_member<_Tp>
 struct __pointer_of<_Tp> {
   using type _LIBCPP_NODEBUG = typename _Tp::pointer;
 };
 
 template <class _Tp>
-  requires(!__has_pointer<_Tp>::value && __has_element_type<_Tp>::value)
+  requires(!__has_pointer_member<_Tp> && __has_element_type_member<_Tp>)
 struct __pointer_of<_Tp> {
   using type _LIBCPP_NODEBUG = typename _Tp::element_type*;
 };
 
 template <class _Tp>
-  requires(!__has_pointer<_Tp>::value && !__has_element_type<_Tp>::value &&
-           __has_element_type<pointer_traits<_Tp>>::value)
+  requires(!__has_pointer_member<_Tp> && !__has_element_type_member<_Tp> &&
+           __has_element_type_member<pointer_traits<_Tp>>)
 struct __pointer_of<_Tp> {
   using type _LIBCPP_NODEBUG = typename pointer_traits<_Tp>::element_type*;
 };
@@ -278,19 +239,8 @@ struct __pointer_of<_Tp> {
 template <typename _Tp>
 using __pointer_of_t _LIBCPP_NODEBUG = typename __pointer_of<_Tp>::type;
 
-template <class _Tp, class _Up>
-struct __pointer_of_or {
-  using type _LIBCPP_NODEBUG = _Up;
-};
-
-template <class _Tp, class _Up>
-  requires requires { typename __pointer_of_t<_Tp>; }
-struct __pointer_of_or<_Tp, _Up> {
-  using type _LIBCPP_NODEBUG = __pointer_of_t<_Tp>;
-};
-
 template <typename _Tp, typename _Up>
-using __pointer_of_or_t _LIBCPP_NODEBUG = typename __pointer_of_or<_Tp, _Up>::type;
+using __pointer_of_or_t _LIBCPP_NODEBUG = __detected_or_t<_Up, __pointer_of_t, _Tp>;
 
 template <class _Smart>
 concept __resettable_smart_pointer = requires(_Smart __s) { __s.reset(); };
-- 
cgit v1.2.3


From 1ab0e7dd60e26ac7c7fc64a273485522f5c5ba02 Mon Sep 17 00:00:00 2001
From: "Jiachen (Yangyang) Wang"
 <130888597+WanderingAura@users.noreply.github.com>
Date: Thu, 19 Jun 2025 09:09:19 +0100
Subject: [LICM] Hoisting writeonly calls (#143799)

Adds support for hoisting `writeonly` calls in LICM.

This patch adds a missing optimization that allows hoisting of
`writeonly` function calls out of loops when it is safe to do so.
Previously, such calls were conservatively retained inside the loop
body, and the redundant calls were only reduced through unrolling,
relying on target-dependent heuristics.

Closes #143267

Testing:
- Modified previously negative tests for hoisting writeonly calls to be
instead positive
- Added test cases for hoisting of two writeonly calls where the
pointers do/do not alias
- Added a test case for not argmemonly writeonly calls.
---
 llvm/lib/Transforms/Scalar/LICM.cpp            | 133 ++++++++++++++----------
 llvm/test/CodeGen/AMDGPU/loop_exit_with_xor.ll |   2 +-
 llvm/test/Transforms/LICM/call-hoisting.ll     | 134 ++++++++++++++++++++++++-
 3 files changed, 211 insertions(+), 58 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index a6bb8b8a21b0..cf84366c4200 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -186,6 +186,9 @@ static bool isSafeToExecuteUnconditionally(
     const Loop *CurLoop, const LoopSafetyInfo *SafetyInfo,
     OptimizationRemarkEmitter *ORE, const Instruction *CtxI,
     AssumptionCache *AC, bool AllowSpeculation);
+static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
+                                    AAResults *AA, Loop *CurLoop,
+                                    SinkAndHoistLICMFlags &Flags);
 static bool pointerInvalidatedByLoop(MemorySSA *MSSA, MemoryUse *MU,
                                      Loop *CurLoop, Instruction &I,
                                      SinkAndHoistLICMFlags &Flags,
@@ -1234,8 +1237,11 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
           /*InvariantGroup=*/false);
     }
 
-    // FIXME: This should use mod/ref information to see if we can hoist or
-    // sink the call.
+    if (Behavior.onlyWritesMemory()) {
+      // can hoist or sink if there are no conflicting read/writes to the
+      // memory location written to by the call.
+      return noConflictingReadWrites(CI, MSSA, AA, CurLoop, Flags);
+    }
 
     return false;
   } else if (auto *FI = dyn_cast<FenceInst>(&I)) {
@@ -1253,57 +1259,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
     // arbitrary number of reads in the loop.
     if (isOnlyMemoryAccess(SI, CurLoop, MSSAU))
       return true;
-    // If there are more accesses than the Promotion cap, then give up as we're
-    // not walking a list that long.
-    if (Flags.tooManyMemoryAccesses())
-      return false;
-
-    auto *SIMD = MSSA->getMemoryAccess(SI);
-    BatchAAResults BAA(*AA);
-    auto *Source = getClobberingMemoryAccess(*MSSA, BAA, Flags, SIMD);
-    // Make sure there are no clobbers inside the loop.
-    if (!MSSA->isLiveOnEntryDef(Source) &&
-           CurLoop->contains(Source->getBlock()))
-      return false;
-
-    // If there are interfering Uses (i.e. their defining access is in the
-    // loop), or ordered loads (stored as Defs!), don't move this store.
-    // Could do better here, but this is conservatively correct.
-    // TODO: Cache set of Uses on the first walk in runOnLoop, update when
-    // moving accesses. Can also extend to dominating uses.
-    for (auto *BB : CurLoop->getBlocks())
-      if (auto *Accesses = MSSA->getBlockAccesses(BB)) {
-        for (const auto &MA : *Accesses)
-          if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
-            auto *MD = getClobberingMemoryAccess(*MSSA, BAA, Flags,
-                const_cast<MemoryUse *>(MU));
-            if (!MSSA->isLiveOnEntryDef(MD) &&
-                CurLoop->contains(MD->getBlock()))
-              return false;
-            // Disable hoisting past potentially interfering loads. Optimized
-            // Uses may point to an access outside the loop, as getClobbering
-            // checks the previous iteration when walking the backedge.
-            // FIXME: More precise: no Uses that alias SI.
-            if (!Flags.getIsSink() && !MSSA->dominates(SIMD, MU))
-              return false;
-          } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
-            if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
-              (void)LI; // Silence warning.
-              assert(!LI->isUnordered() && "Expected unordered load");
-              return false;
-            }
-            // Any call, while it may not be clobbering SI, it may be a use.
-            if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
-              // Check if the call may read from the memory location written
-              // to by SI. Check CI's attributes and arguments; the number of
-              // such checks performed is limited above by NoOfMemAccTooLarge.
-              ModRefInfo MRI = BAA.getModRefInfo(CI, MemoryLocation::get(SI));
-              if (isModOrRefSet(MRI))
-                return false;
-            }
-          }
-      }
-    return true;
+    return noConflictingReadWrites(SI, MSSA, AA, CurLoop, Flags);
   }
 
   assert(!I.mayReadOrWriteMemory() && "unhandled aliasing");
@@ -2330,6 +2286,77 @@ collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
   return Result;
 }
 
+// For a given store instruction or writeonly call instruction, this function
+// checks that there are no read or writes that conflict with the memory
+// access in the instruction
+static bool noConflictingReadWrites(Instruction *I, MemorySSA *MSSA,
+                                    AAResults *AA, Loop *CurLoop,
+                                    SinkAndHoistLICMFlags &Flags) {
+  assert(isa<CallInst>(*I) || isa<StoreInst>(*I));
+  // If there are more accesses than the Promotion cap, then give up as we're
+  // not walking a list that long.
+  if (Flags.tooManyMemoryAccesses())
+    return false;
+
+  auto *IMD = MSSA->getMemoryAccess(I);
+  BatchAAResults BAA(*AA);
+  auto *Source = getClobberingMemoryAccess(*MSSA, BAA, Flags, IMD);
+  // Make sure there are no clobbers inside the loop.
+  if (!MSSA->isLiveOnEntryDef(Source) && CurLoop->contains(Source->getBlock()))
+    return false;
+
+  // If there are interfering Uses (i.e. their defining access is in the
+  // loop), or ordered loads (stored as Defs!), don't move this store.
+  // Could do better here, but this is conservatively correct.
+  // TODO: Cache set of Uses on the first walk in runOnLoop, update when
+  // moving accesses. Can also extend to dominating uses.
+  for (auto *BB : CurLoop->getBlocks()) {
+    auto *Accesses = MSSA->getBlockAccesses(BB);
+    if (!Accesses)
+      continue;
+    for (const auto &MA : *Accesses)
+      if (const auto *MU = dyn_cast<MemoryUse>(&MA)) {
+        auto *MD = getClobberingMemoryAccess(*MSSA, BAA, Flags,
+                                             const_cast<MemoryUse *>(MU));
+        if (!MSSA->isLiveOnEntryDef(MD) && CurLoop->contains(MD->getBlock()))
+          return false;
+        // Disable hoisting past potentially interfering loads. Optimized
+        // Uses may point to an access outside the loop, as getClobbering
+        // checks the previous iteration when walking the backedge.
+        // FIXME: More precise: no Uses that alias I.
+        if (!Flags.getIsSink() && !MSSA->dominates(IMD, MU))
+          return false;
+      } else if (const auto *MD = dyn_cast<MemoryDef>(&MA)) {
+        if (auto *LI = dyn_cast<LoadInst>(MD->getMemoryInst())) {
+          (void)LI; // Silence warning.
+          assert(!LI->isUnordered() && "Expected unordered load");
+          return false;
+        }
+        // Any call, while it may not be clobbering I, it may be a use.
+        if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
+          // Check if the call may read from the memory location written
+          // to by I. Check CI's attributes and arguments; the number of
+          // such checks performed is limited above by NoOfMemAccTooLarge.
+          if (auto *SI = dyn_cast<StoreInst>(I)) {
+            ModRefInfo MRI = BAA.getModRefInfo(CI, MemoryLocation::get(SI));
+            if (isModOrRefSet(MRI))
+              return false;
+          } else {
+            auto *SCI = cast<CallInst>(I);
+            // If the instruction we are wanting to hoist is also a call
+            // instruction then we need not check mod/ref info with itself
+            if (SCI == CI)
+              continue;
+            ModRefInfo MRI = BAA.getModRefInfo(CI, SCI);
+            if (isModOrRefSet(MRI))
+              return false;
+          }
+        }
+      }
+  }
+  return true;
+}
+
 static bool pointerInvalidatedByLoop(MemorySSA *MSSA, MemoryUse *MU,
                                      Loop *CurLoop, Instruction &I,
                                      SinkAndHoistLICMFlags &Flags,
diff --git a/llvm/test/CodeGen/AMDGPU/loop_exit_with_xor.ll b/llvm/test/CodeGen/AMDGPU/loop_exit_with_xor.ll
index e37dcf60506b..2864e0554a27 100644
--- a/llvm/test/CodeGen/AMDGPU/loop_exit_with_xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop_exit_with_xor.ll
@@ -64,6 +64,7 @@ define void @doesnt_need_and(i32 %arg) {
 ; GCN-LABEL: doesnt_need_and:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], s4
 ; GCN-NEXT:    s_mov_b32 s6, 0
 ; GCN-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-NEXT:  .LBB1_1: ; %loop
@@ -71,7 +72,6 @@ define void @doesnt_need_and(i32 %arg) {
 ; GCN-NEXT:    s_add_i32 s6, s6, 1
 ; GCN-NEXT:    v_cmp_le_u32_e32 vcc, s6, v0
 ; GCN-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN-NEXT:    buffer_store_dword v0, off, s[4:7], s4
 ; GCN-NEXT:    s_andn2_b64 exec, exec, s[4:5]
 ; GCN-NEXT:    s_cbranch_execnz .LBB1_1
 ; GCN-NEXT:  ; %bb.2: ; %loopexit
diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll
index 7124b4e445eb..aa8c8bbed550 100644
--- a/llvm/test/Transforms/LICM/call-hoisting.ll
+++ b/llvm/test/Transforms/LICM/call-hoisting.ll
@@ -118,14 +118,16 @@ exit:
 
 declare void @store(i32 %val, ptr %p) argmemonly writeonly nounwind
 
+; loop invariant calls to writeonly functions such as the above
+; should be hoisted
 define void @test(ptr %loc) {
 ; CHECK-LABEL: define void @test(
 ; CHECK-SAME: ptr [[LOC:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
@@ -150,10 +152,10 @@ define void @test_multiexit(ptr %loc, i1 %earlycnd) {
 ; CHECK-LABEL: define void @test_multiexit(
 ; CHECK-SAME: ptr [[LOC:%.*]], i1 [[EARLYCND:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[BACKEDGE:.*]] ]
-; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    br i1 [[EARLYCND]], label %[[EXIT1:.*]], label %[[BACKEDGE]]
 ; CHECK:       [[BACKEDGE]]:
@@ -183,6 +185,97 @@ exit2:
   ret void
 }
 
+; cannot be hoisted because the two pointers can alias one another
+define void @neg_two_pointer(ptr %loc, ptr %otherloc) {
+; CHECK-LABEL: define void @neg_two_pointer(
+; CHECK-SAME: ptr [[LOC:%.*]], ptr [[OTHERLOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    call void @store(i32 1, ptr [[OTHERLOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, ptr %loc)
+  call void @store(i32 1, ptr %otherloc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; hoisted due to pointers not aliasing
+define void @two_pointer_noalias(ptr noalias %loc, ptr noalias %otherloc) {
+; CHECK-LABEL: define void @two_pointer_noalias(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[OTHERLOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    call void @store(i32 1, ptr [[OTHERLOC]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @store(i32 0, ptr %loc)
+  call void @store(i32 1, ptr %otherloc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
+; when there's a conflicting read, store call should not be hoisted
+define void @neg_conflicting_read(ptr noalias %loc, ptr noalias %otherloc) {
+; CHECK-LABEL: define void @neg_conflicting_read(
+; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[OTHERLOC:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT:    call void @load(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    call void @store(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  call void @store(i32 0, ptr %loc)
+  br label %loop
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @load(i32 0, ptr %loc)
+  call void @store(i32 0, ptr %loc)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+exit:
+  ret void
+}
+
 define void @neg_lv_value(ptr %loc) {
 ; CHECK-LABEL: define void @neg_lv_value(
 ; CHECK-SAME: ptr [[LOC:%.*]]) {
@@ -406,14 +499,47 @@ exit:
   ret void
 }
 
-define void @neg_not_argmemonly(ptr %loc) {
+; when the call is not argmemonly and is not the only memory access we
+; do not hoist
+define void @neg_not_argmemonly(ptr %loc, ptr %loc2) {
 ; CHECK-LABEL: define void @neg_not_argmemonly(
-; CHECK-SAME: ptr [[LOC:%.*]]) {
+; CHECK-SAME: ptr [[LOC:%.*]], ptr [[LOC2:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    call void @not_argmemonly(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    call void @load(i32 0, ptr [[LOC2]])
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [0, %entry], [%iv.next, %loop]
+  call void @not_argmemonly(i32 0, ptr %loc)
+  call void @load(i32 0, ptr %loc2)
+  %iv.next = add i32 %iv, 1
+  %cmp = icmp slt i32 %iv, 200
+  br i1 %cmp, label %loop, label %exit
+
+exit:
+  ret void
+}
+
+; when the call is not argmemonly and is only memory access we hoist it
+define void @not_argmemonly_hoisted(ptr %loc, ptr %loc2) {
+; CHECK-LABEL: define void @not_argmemonly_hoisted(
+; CHECK-SAME: ptr [[LOC:%.*]], ptr [[LOC2:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    call void @not_argmemonly(i32 0, ptr [[LOC]])
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
 ; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 200
 ; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]]
-- 
cgit v1.2.3


From 0e1aab1ec833d7f8e9897b0940c634385036fdee Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 19 Jun 2025 10:13:55 +0200
Subject: [AMDGPU] Improve test coverage for G_INTTOPTR and G_PTRTOINT
 (#142603)

Test P0 through P6 + P8 for both S/VGPRs.
---
 .../AMDGPU/GlobalISel/regbankselect-inttoptr.mir   | 236 ++++++++++++++++++++-
 .../AMDGPU/GlobalISel/regbankselect-ptrtoint.mir   | 232 +++++++++++++++++++-
 2 files changed, 458 insertions(+), 10 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
index 053aede615f8..42600d7d0dd7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
@@ -3,29 +3,29 @@
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: inttoptr_s
+name: inttoptr_s_p0
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0_sgpr1
-    ; CHECK-LABEL: name: inttoptr_s
+    ; CHECK-LABEL: name: inttoptr_s_p0
     ; CHECK: liveins: $sgpr0_sgpr1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
-    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p4) = G_INTTOPTR [[COPY]](s64)
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p0) = G_INTTOPTR [[COPY]](s64)
     %0:_(s64) = COPY $sgpr0_sgpr1
-    %1:_(p4) = G_INTTOPTR %0
+    %1:_(p0) = G_INTTOPTR %0
 ...
 
 ---
-name: inttoptr_v
+name: inttoptr_v_p0
 legalized: true
 
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
-    ; CHECK-LABEL: name: inttoptr_v
+    ; CHECK-LABEL: name: inttoptr_v_p0
     ; CHECK: liveins: $vgpr0_vgpr1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
@@ -33,3 +33,227 @@ body: |
     %0:_(s64) = COPY $vgpr0_vgpr1
     %1:_(p0) = G_INTTOPTR %0
 ...
+
+---
+name: inttoptr_s_p1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: inttoptr_s_p1
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p1) = G_INTTOPTR [[COPY]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(p1) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: inttoptr_v_p1
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p1) = G_INTTOPTR [[COPY]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p1) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p2
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: inttoptr_s_p2
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p2) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(p2) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p2
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: inttoptr_v_p2
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p2) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p2) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p3
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: inttoptr_s_p3
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p3) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(p3) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p3
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: inttoptr_v_p3
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p3) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p3) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p4
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: inttoptr_s_p4
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s64) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p4) = G_INTTOPTR [[COPY]](s64)
+    %0:_(s64) = COPY $sgpr0_sgpr1
+    %1:_(p4) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p4
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: inttoptr_v_p4
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s64) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p4) = G_INTTOPTR [[COPY]](s64)
+    %0:_(s64) = COPY $vgpr0_vgpr1
+    %1:_(p4) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p5
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: inttoptr_s_p5
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p5) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(p5) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p5
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: inttoptr_v_p5
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p5) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p5) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p6
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: inttoptr_s_p6
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s32) = COPY $sgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p6) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $sgpr0
+    %1:_(p6) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p6
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: inttoptr_v_p6
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p6) = G_INTTOPTR [[COPY]](s32)
+    %0:_(s32) = COPY $vgpr0
+    %1:_(p6) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_s_p8
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-LABEL: name: inttoptr_s_p8
+    ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(s128) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:sgpr(p8) = G_INTTOPTR [[COPY]](s128)
+    %0:_(s128) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+    %1:_(p8) = G_INTTOPTR %0
+...
+
+---
+name: inttoptr_v_p8
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-LABEL: name: inttoptr_v_p8
+    ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:vgpr(p8) = G_INTTOPTR [[COPY]](s128)
+    %0:_(s128) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(p8) = G_INTTOPTR %0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
index 2db1a5b1d2ba..9240c9f6df40 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
@@ -3,13 +3,45 @@
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
 
 ---
-name: ptrtoint_s
+name: ptrtoint_s_p0
 legalized: true
 
 body: |
   bb.0:
     liveins: $sgpr0_sgpr1
-    ; CHECK-LABEL: name: ptrtoint_s
+    ; CHECK-LABEL: name: ptrtoint_s_p0
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p0) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s64) = G_PTRTOINT [[COPY]](p0)
+    %0:_(p0) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p0
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ptrtoint_v_p0
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p0) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s64) = G_PTRTOINT [[COPY]](p0)
+    %0:_(p0) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p1
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: ptrtoint_s_p1
     ; CHECK: liveins: $sgpr0_sgpr1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p1) = COPY $sgpr0_sgpr1
@@ -19,13 +51,13 @@ body: |
 ...
 
 ---
-name: ptrtoint_v
+name: ptrtoint_v_p1
 legalized: true
 
 body: |
   bb.0:
     liveins: $vgpr0_vgpr1
-    ; CHECK-LABEL: name: ptrtoint_v
+    ; CHECK-LABEL: name: ptrtoint_v_p1
     ; CHECK: liveins: $vgpr0_vgpr1
     ; CHECK-NEXT: {{  $}}
     ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p1) = COPY $vgpr0_vgpr1
@@ -33,3 +65,195 @@ body: |
     %0:_(p1) = COPY $vgpr0_vgpr1
     %1:_(s64) = G_PTRTOINT %0
 ...
+
+---
+name: ptrtoint_s_p2
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ptrtoint_s_p2
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p2) = COPY $sgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s32) = G_PTRTOINT [[COPY]](p2)
+    %0:_(p2) = COPY $sgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p2
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ptrtoint_v_p2
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p2) = COPY $vgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s32) = G_PTRTOINT [[COPY]](p2)
+    %0:_(p2) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p3
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ptrtoint_s_p3
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p3) = COPY $sgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s32) = G_PTRTOINT [[COPY]](p3)
+    %0:_(p3) = COPY $sgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p3
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ptrtoint_v_p3
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s32) = G_PTRTOINT [[COPY]](p3)
+    %0:_(p3) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p4
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1
+    ; CHECK-LABEL: name: ptrtoint_s_p4
+    ; CHECK: liveins: $sgpr0_sgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p4) = COPY $sgpr0_sgpr1
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s64) = G_PTRTOINT [[COPY]](p4)
+    %0:_(p4) = COPY $sgpr0_sgpr1
+    %1:_(s64) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p4
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1
+    ; CHECK-LABEL: name: ptrtoint_v_p4
+    ; CHECK: liveins: $vgpr0_vgpr1
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p4) = COPY $vgpr0_vgpr1
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s64) = G_PTRTOINT [[COPY]](p4)
+    %0:_(p4) = COPY $vgpr0_vgpr1
+    %1:_(s64) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p5
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ptrtoint_s_p5
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p5) = COPY $sgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s32) = G_PTRTOINT [[COPY]](p5)
+    %0:_(p5) = COPY $sgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p5
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ptrtoint_v_p5
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s32) = G_PTRTOINT [[COPY]](p5)
+    %0:_(p5) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p6
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0
+    ; CHECK-LABEL: name: ptrtoint_s_p6
+    ; CHECK: liveins: $sgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p6) = COPY $sgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s32) = G_PTRTOINT [[COPY]](p6)
+    %0:_(p6) = COPY $sgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p6
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0
+    ; CHECK-LABEL: name: ptrtoint_v_p6
+    ; CHECK: liveins: $vgpr0
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p6) = COPY $vgpr0
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s32) = G_PTRTOINT [[COPY]](p6)
+    %0:_(p6) = COPY $vgpr0
+    %1:_(s32) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_s_p8
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-LABEL: name: ptrtoint_s_p8
+    ; CHECK: liveins: $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr(p8) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:sgpr(s128) = G_PTRTOINT [[COPY]](p8)
+    %0:_(p8) = COPY $sgpr0_sgpr1_sgpr2_sgpr3
+    %1:_(s128) = G_PTRTOINT %0
+...
+
+---
+name: ptrtoint_v_p8
+legalized: true
+
+body: |
+  bb.0:
+    liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-LABEL: name: ptrtoint_v_p8
+    ; CHECK: liveins: $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:vgpr(p8) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    ; CHECK-NEXT: [[PTRTOINT:%[0-9]+]]:vgpr(s128) = G_PTRTOINT [[COPY]](p8)
+    %0:_(p8) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
+    %1:_(s128) = G_PTRTOINT %0
+...
-- 
cgit v1.2.3


From 62fe5e428acc2c5ef9b144c5737d55b17b55feac Mon Sep 17 00:00:00 2001
From: Sameer Sahasrabuddhe <sameer.sahasrabuddhe@amd.com>
Date: Thu, 19 Jun 2025 13:47:37 +0530
Subject: [NFC][AMDGPU] print more info when debugging SIInsertWaitcnts pass
 (#144629)

---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 112 ++++++++++++++++++++--------
 1 file changed, 80 insertions(+), 32 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index ca8e3244edd1..f7b88bf2d5eb 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -104,25 +104,38 @@ struct HardwareLimits {
   unsigned KmcntMax;     // gfx12+ only.
 };
 
+#define AMDGPU_DECLARE_WAIT_EVENTS(DECL)                                       \
+  DECL(VMEM_ACCESS)              /* vmem read & write */                       \
+  DECL(VMEM_READ_ACCESS)         /* vmem read */                               \
+  DECL(VMEM_SAMPLER_READ_ACCESS) /* vmem SAMPLER read (gfx12+ only) */         \
+  DECL(VMEM_BVH_READ_ACCESS)     /* vmem BVH read (gfx12+ only) */             \
+  DECL(VMEM_WRITE_ACCESS)        /* vmem write that is not scratch */          \
+  DECL(SCRATCH_WRITE_ACCESS)     /* vmem write that may be scratch */          \
+  DECL(LDS_ACCESS)               /* lds read & write */                        \
+  DECL(GDS_ACCESS)               /* gds read & write */                        \
+  DECL(SQ_MESSAGE)               /* send message */                            \
+  DECL(SMEM_ACCESS)              /* scalar-memory read & write */              \
+  DECL(EXP_GPR_LOCK)             /* export holding on its data src */          \
+  DECL(GDS_GPR_LOCK)             /* GDS holding on its data and addr src */    \
+  DECL(EXP_POS_ACCESS)           /* write to export position */                \
+  DECL(EXP_PARAM_ACCESS)         /* write to export parameter */               \
+  DECL(VMW_GPR_LOCK)             /* vmem write holding on its data src */      \
+  DECL(EXP_LDS_ACCESS)           /* read by ldsdir counting as export */
+
+// clang-format off
+#define AMDGPU_EVENT_ENUM(Name) Name,
 enum WaitEventType {
-  VMEM_ACCESS,              // vector-memory read & write
-  VMEM_READ_ACCESS,         // vector-memory read
-  VMEM_SAMPLER_READ_ACCESS, // vector-memory SAMPLER read (gfx12+ only)
-  VMEM_BVH_READ_ACCESS,     // vector-memory BVH read (gfx12+ only)
-  VMEM_WRITE_ACCESS,        // vector-memory write that is not scratch
-  SCRATCH_WRITE_ACCESS,     // vector-memory write that may be scratch
-  LDS_ACCESS,               // lds read & write
-  GDS_ACCESS,               // gds read & write
-  SQ_MESSAGE,               // send message
-  SMEM_ACCESS,              // scalar-memory read & write
-  EXP_GPR_LOCK,             // export holding on its data src
-  GDS_GPR_LOCK,             // GDS holding on its data and addr src
-  EXP_POS_ACCESS,           // write to export position
-  EXP_PARAM_ACCESS,         // write to export parameter
-  VMW_GPR_LOCK,             // vector-memory write holding on its data src
-  EXP_LDS_ACCESS,           // read by ldsdir counting as export
-  NUM_WAIT_EVENTS,
+  AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_ENUM)
+  NUM_WAIT_EVENTS
 };
+#undef AMDGPU_EVENT_ENUM
+
+#define AMDGPU_EVENT_NAME(Name) #Name,
+static constexpr StringLiteral WaitEventTypeName[] = {
+  AMDGPU_DECLARE_WAIT_EVENTS(AMDGPU_EVENT_NAME)
+};
+#undef AMDGPU_EVENT_NAME
+// clang-format on
 
 // The mapping is:
 //  0                .. SQ_MAX_PGM_VGPRS-1               real VGPRs
@@ -1100,6 +1113,20 @@ void WaitcntBrackets::print(raw_ostream &OS) const {
     }
     OS << '\n';
   }
+
+  OS << "Pending Events: ";
+  if (hasPendingEvent()) {
+    ListSeparator LS;
+    for (unsigned I = 0; I != NUM_WAIT_EVENTS; ++I) {
+      if (hasPendingEvent((WaitEventType)I)) {
+        OS << LS << WaitEventTypeName[I];
+      }
+    }
+  } else {
+    OS << "none";
+  }
+  OS << '\n';
+
   OS << '\n';
 }
 
@@ -1265,10 +1292,15 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
   MachineInstr *WaitcntInstr = nullptr;
   MachineInstr *WaitcntVsCntInstr = nullptr;
 
+  LLVM_DEBUG(dbgs() << "PreGFX12::applyPreexistingWaitcnt at: " << *It);
+
   for (auto &II :
        make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
-    if (II.isMetaInstruction())
+    LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
+    if (II.isMetaInstruction()) {
+      LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
       continue;
+    }
 
     unsigned Opcode = SIInstrInfo::getNonSoftWaitcntOpcode(II.getOpcode());
     bool TrySimplify = Opcode != II.getOpcode() && !OptNone;
@@ -1320,9 +1352,9 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
 
     LLVM_DEBUG(It == WaitcntInstr->getParent()->end()
                    ? dbgs()
-                         << "applyPreexistingWaitcnt\n"
+                         << "applied pre-existing waitcnt\n"
                          << "New Instr at block end: " << *WaitcntInstr << '\n'
-                   : dbgs() << "applyPreexistingWaitcnt\n"
+                   : dbgs() << "applied pre-existing waitcnt\n"
                             << "Old Instr: " << *It
                             << "New Instr: " << *WaitcntInstr << '\n');
   }
@@ -1336,10 +1368,10 @@ bool WaitcntGeneratorPreGFX12::applyPreexistingWaitcnt(
     Wait.StoreCnt = ~0u;
 
     LLVM_DEBUG(It == WaitcntVsCntInstr->getParent()->end()
-                   ? dbgs() << "applyPreexistingWaitcnt\n"
+                   ? dbgs() << "applied pre-existing waitcnt\n"
                             << "New Instr at block end: " << *WaitcntVsCntInstr
                             << '\n'
-                   : dbgs() << "applyPreexistingWaitcnt\n"
+                   : dbgs() << "applied pre-existing waitcnt\n"
                             << "Old Instr: " << *It
                             << "New Instr: " << *WaitcntVsCntInstr << '\n');
   }
@@ -1413,10 +1445,15 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
   MachineInstr *CombinedStoreDsCntInstr = nullptr;
   MachineInstr *WaitInstrs[NUM_EXTENDED_INST_CNTS] = {};
 
+  LLVM_DEBUG(dbgs() << "GFX12Plus::applyPreexistingWaitcnt at: " << *It);
+
   for (auto &II :
        make_early_inc_range(make_range(OldWaitcntInstr.getIterator(), It))) {
-    if (II.isMetaInstruction())
+    LLVM_DEBUG(dbgs() << "pre-existing iter: " << II);
+    if (II.isMetaInstruction()) {
+      LLVM_DEBUG(dbgs() << "skipped meta instruction\n");
       continue;
+    }
 
     MachineInstr **UpdatableInstr;
 
@@ -1486,10 +1523,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
       Wait.DsCnt = ~0u;
 
       LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
-                     ? dbgs() << "applyPreexistingWaitcnt\n"
+                     ? dbgs() << "applied pre-existing waitcnt\n"
                               << "New Instr at block end: "
                               << *CombinedLoadDsCntInstr << '\n'
-                     : dbgs() << "applyPreexistingWaitcnt\n"
+                     : dbgs() << "applied pre-existing waitcnt\n"
                               << "Old Instr: " << *It << "New Instr: "
                               << *CombinedLoadDsCntInstr << '\n');
     } else {
@@ -1511,10 +1548,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
       Wait.DsCnt = ~0u;
 
       LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
-                     ? dbgs() << "applyPreexistingWaitcnt\n"
+                     ? dbgs() << "applied pre-existing waitcnt\n"
                               << "New Instr at block end: "
                               << *CombinedStoreDsCntInstr << '\n'
-                     : dbgs() << "applyPreexistingWaitcnt\n"
+                     : dbgs() << "applied pre-existing waitcnt\n"
                               << "Old Instr: " << *It << "New Instr: "
                               << *CombinedStoreDsCntInstr << '\n');
     } else {
@@ -1570,10 +1607,10 @@ bool WaitcntGeneratorGFX12Plus::applyPreexistingWaitcnt(
       setNoWait(Wait, CT);
 
       LLVM_DEBUG(It == OldWaitcntInstr.getParent()->end()
-                     ? dbgs() << "applyPreexistingWaitcnt\n"
+                     ? dbgs() << "applied pre-existing waitcnt\n"
                               << "New Instr at block end: " << *WaitInstrs[CT]
                               << '\n'
-                     : dbgs() << "applyPreexistingWaitcnt\n"
+                     : dbgs() << "applied pre-existing waitcnt\n"
                               << "Old Instr: " << *It
                               << "New Instr: " << *WaitInstrs[CT] << '\n');
     } else {
@@ -2306,7 +2343,8 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
   bool Modified = false;
 
   LLVM_DEBUG({
-    dbgs() << "*** Block" << Block.getNumber() << " ***";
+    dbgs() << "*** Begin Block: ";
+    Block.printName(dbgs());
     ScoreBrackets.dump();
   });
 
@@ -2437,6 +2475,12 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF,
   Modified |= generateWaitcnt(Wait, Block.instr_end(), Block, ScoreBrackets,
                               OldWaitcntInstr);
 
+  LLVM_DEBUG({
+    dbgs() << "*** End Block: ";
+    Block.printName(dbgs());
+    ScoreBrackets.dump();
+  });
+
   return Modified;
 }
 
@@ -2699,8 +2743,10 @@ bool SIInsertWaitcnts::run(MachineFunction &MF) {
           BlockInfo &SuccBI = SuccBII->second;
           if (!SuccBI.Incoming) {
             SuccBI.Dirty = true;
-            if (SuccBII <= BII)
+            if (SuccBII <= BII) {
+              LLVM_DEBUG(dbgs() << "repeat on backedge\n");
               Repeat = true;
+            }
             if (!MoveBracketsToSucc) {
               MoveBracketsToSucc = &SuccBI;
             } else {
@@ -2708,8 +2754,10 @@ bool SIInsertWaitcnts::run(MachineFunction &MF) {
             }
           } else if (SuccBI.Incoming->merge(*Brackets)) {
             SuccBI.Dirty = true;
-            if (SuccBII <= BII)
+            if (SuccBII <= BII) {
+              LLVM_DEBUG(dbgs() << "repeat on backedge\n");
               Repeat = true;
+            }
           }
         }
         if (MoveBracketsToSucc)
-- 
cgit v1.2.3


From 8f82c027c8969d965c43909da639e7790af19956 Mon Sep 17 00:00:00 2001
From: Pierre van Houtryve <pierre.vanhoutryve@amd.com>
Date: Thu, 19 Jun 2025 10:21:17 +0200
Subject: [AMDGPU] New RegBankSelect: Add rules for `G_PTRTOINT` and
 `G_INTTOPTR` (#142604)

---
 llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp    | 16 +++++++++++++++-
 .../CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir |  1 +
 .../CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir |  1 +
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index 5402129e4188..db3f6bd360b3 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -718,7 +718,21 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
       .Any({{DivP1}, {{VgprP1}, {VgprP1, Vgpr64}}})
       .Any({{DivP0}, {{VgprP0}, {VgprP0, Vgpr64}}});
 
-  addRulesForGOpcs({G_INTTOPTR}).Any({{UniP4}, {{SgprP4}, {Sgpr64}}});
+  addRulesForGOpcs({G_INTTOPTR})
+      .Any({{UniPtr32}, {{SgprPtr32}, {Sgpr32}}})
+      .Any({{DivPtr32}, {{VgprPtr32}, {Vgpr32}}})
+      .Any({{UniPtr64}, {{SgprPtr64}, {Sgpr64}}})
+      .Any({{DivPtr64}, {{VgprPtr64}, {Vgpr64}}})
+      .Any({{UniPtr128}, {{SgprPtr128}, {Sgpr128}}})
+      .Any({{DivPtr128}, {{VgprPtr128}, {Vgpr128}}});
+
+  addRulesForGOpcs({G_PTRTOINT})
+      .Any({{UniS32}, {{Sgpr32}, {SgprPtr32}}})
+      .Any({{DivS32}, {{Vgpr32}, {VgprPtr32}}})
+      .Any({{UniS64}, {{Sgpr64}, {SgprPtr64}}})
+      .Any({{DivS64}, {{Vgpr64}, {VgprPtr64}}})
+      .Any({{UniS128}, {{Sgpr128}, {SgprPtr128}}})
+      .Any({{DivS128}, {{Vgpr128}, {VgprPtr128}}});
 
   addRulesForGOpcs({G_ABS}, Standard).Uni(S16, {{Sgpr32Trunc}, {Sgpr32SExt}});
 
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
index 42600d7d0dd7..aec68e5a5b71 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-inttoptr.mir
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -verify-machineinstrs -o - | FileCheck %s
 
 ---
 name: inttoptr_s_p0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
index 9240c9f6df40..31cbae9ab47f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankselect-ptrtoint.mir
@@ -1,6 +1,7 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-fast | FileCheck %s
 # RUN: llc -mtriple=amdgcn -run-pass=regbankselect %s -verify-machineinstrs -o - -regbankselect-greedy | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=fiji -run-pass="amdgpu-regbankselect,amdgpu-regbanklegalize" %s -verify-machineinstrs -o - | FileCheck %s
 
 ---
 name: ptrtoint_s_p0
-- 
cgit v1.2.3


From 681db064d221e9eef024ce0aef6165caa37fbfd2 Mon Sep 17 00:00:00 2001
From: Kunqiu Chen <camsyn@foxmail.com>
Date: Thu, 19 Jun 2025 16:25:51 +0800
Subject: [TSan] Make Shadow/Meta region inclusive-exclusive (#144647)

This commit changes the interval shadow/meta address check from
inclusive-inclusive ( $[\mathrm{start}, \mathrm{end}]$ ) to
inclusive-exclusive ( $[\mathrm{start}, \mathrm{end})$ ), to resolve the
ambiguity of the end point address. This also aligns the logic with the
check for `isAppMem` (i.e., inclusive-exclusive), ensuring consistent
behavior across all memory classifications.

1. The `isShadowMem` and `isMetaMem` checks previously used an
inclusive-inclusive interval, i.e., $[\mathrm{start}, \mathrm{end}]$,
which could lead to a boundary address being incorrectly classified as
both Shadow and Meta memory, e.g., 0x3000_0000_0000 in
`Mapping48AddressSpace`.
- What's more, even when Shadow doesn't border Meta, `ShadowMem::end`
cannot be considered a legal shadow address, as TSan protects the gap,
i.e., `ProtectRange(ShadowEnd(), MetaShadowBeg());`

2. `ShadowMem`/`MetaMem` addresses are derived from `AppMem` using an
affine-like transformation (`* factor + bias`). This transformation
includes two extra modifications: high- and low-order bits are masked
out, and for Shadow Memory, an optional XOR operation may be applied to
prevent conflicts with certain AppMem regions.
- Given that all AppMem regions are defined as inclusive-exclusive
intervals, $[\mathrm{start}, \mathrm{end})$, the resulting Shadow/Meta
regions should logically also be inclusive-exclusive.

Note: This change is purely for improving code consistency and should
have no functional impact. In practice, the exact endpoint addresses of
the Shadow/Meta regions are generally not reached.
---
 compiler-rt/lib/tsan/rtl/tsan_platform.h     | 4 ++--
 compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/compiler-rt/lib/tsan/rtl/tsan_platform.h b/compiler-rt/lib/tsan/rtl/tsan_platform.h
index 354f6da6a64a..ada594bc11fc 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_platform.h
+++ b/compiler-rt/lib/tsan/rtl/tsan_platform.h
@@ -931,7 +931,7 @@ bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
 struct IsShadowMemImpl {
   template <typename Mapping>
   static bool Apply(uptr mem) {
-    return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
+    return mem >= Mapping::kShadowBeg && mem < Mapping::kShadowEnd;
   }
 };
 
@@ -943,7 +943,7 @@ bool IsShadowMem(RawShadow *p) {
 struct IsMetaMemImpl {
   template <typename Mapping>
   static bool Apply(uptr mem) {
-    return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
+    return mem >= Mapping::kMetaShadowBeg && mem < Mapping::kMetaShadowEnd;
   }
 };
 
diff --git a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
index cf07686d968d..dbdc6359d92a 100644
--- a/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
+++ b/compiler-rt/lib/tsan/rtl/tsan_rtl_access.cpp
@@ -525,7 +525,7 @@ SECOND:
 void ShadowSet(RawShadow* p, RawShadow* end, RawShadow v) {
   DCHECK_LE(p, end);
   DCHECK(IsShadowMem(p));
-  DCHECK(IsShadowMem(end));
+  DCHECK(p == end || IsShadowMem(end - 1));
   UNUSED const uptr kAlign = kShadowCnt * kShadowSize;
   DCHECK_EQ(reinterpret_cast<uptr>(p) % kAlign, 0);
   DCHECK_EQ(reinterpret_cast<uptr>(end) % kAlign, 0);
@@ -669,7 +669,7 @@ void MemoryAccessRangeT(ThreadState* thr, uptr pc, uptr addr, uptr size) {
   RawShadow* shadow_mem = MemToShadow(addr);
   DPrintf2("#%d: MemoryAccessRange: @%p %p size=%d is_read=%d\n", thr->tid,
            (void*)pc, (void*)addr, (int)size, is_read);
-
+  DCHECK_NE(size, 0);
 #if SANITIZER_DEBUG
   if (!IsAppMem(addr)) {
     Printf("Access to non app mem start: %p\n", (void*)addr);
-- 
cgit v1.2.3


From 584cc376870505821b5ff0b0e80be85ee563ff0c Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 19 Jun 2025 10:37:18 +0200
Subject: [libc++] Move std::abs into __math/abs.h (#139586)

`template <class = int>` is also added to our implementations to avoid
an ambiguity between the libc's version and our version when both are
visible.

This avoids including `<stdlib.h>` in `<math.h>`.
---
 libcxx/include/__math/abs.h                    | 24 ++++++++++++++++++++++++
 libcxx/include/math.h                          | 11 +----------
 libcxx/include/stdlib.h                        | 19 ++-----------------
 libcxx/test/std/numerics/c.math/abs.verify.cpp |  4 ++--
 4 files changed, 29 insertions(+), 29 deletions(-)

diff --git a/libcxx/include/__math/abs.h b/libcxx/include/__math/abs.h
index fc3bf3a2c7c3..b780159f11eb 100644
--- a/libcxx/include/__math/abs.h
+++ b/libcxx/include/__math/abs.h
@@ -39,6 +39,30 @@ template <class _A1, __enable_if_t<is_integral<_A1>::value, int> = 0>
   return __builtin_fabs((double)__x);
 }
 
+// abs
+
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline float abs(float __x) _NOEXCEPT { return __builtin_fabsf(__x); }
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline double abs(double __x) _NOEXCEPT { return __builtin_fabs(__x); }
+
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline long double abs(long double __x) _NOEXCEPT {
+  return __builtin_fabsl(__x);
+}
+
+template <class = int>
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline int abs(int __x) _NOEXCEPT {
+  return __builtin_abs(__x);
+}
+
+template <class = int>
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline long abs(long __x) _NOEXCEPT {
+  return __builtin_labs(__x);
+}
+
+template <class = int>
+[[__nodiscard__]] _LIBCPP_HIDE_FROM_ABI inline long long abs(long long __x) _NOEXCEPT {
+  return __builtin_llabs(__x);
+}
+
 } // namespace __math
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/math.h b/libcxx/include/math.h
index de2dacde282c..929bef638504 100644
--- a/libcxx/include/math.h
+++ b/libcxx/include/math.h
@@ -378,9 +378,7 @@ extern "C++" {
 #      include <__math/traits.h>
 #      include <__math/trigonometric_functions.h>
 #      include <__type_traits/enable_if.h>
-#      include <__type_traits/is_floating_point.h>
 #      include <__type_traits/is_integral.h>
-#      include <stdlib.h>
 
 // fpclassify relies on implementation-defined constants, so we can't move it to a detail header
 _LIBCPP_BEGIN_NAMESPACE_STD
@@ -431,19 +429,12 @@ using std::__math::isnormal;
 using std::__math::isunordered;
 #      endif // _LIBCPP_MSVCRT
 
-// abs
-//
-// handled in stdlib.h
-
-// div
-//
-// handled in stdlib.h
-
 // We have to provide double overloads for <math.h> to work on platforms that don't provide the full set of math
 // functions. To make the overload set work with multiple functions that take the same arguments, we make our overloads
 // templates. Functions are preferred over function templates during overload resolution, which means that our overload
 // will only be selected when the C library doesn't provide one.
 
+using std::__math::abs;
 using std::__math::acos;
 using std::__math::acosh;
 using std::__math::asin;
diff --git a/libcxx/include/stdlib.h b/libcxx/include/stdlib.h
index 39550f36bb6e..8dfdfa416f08 100644
--- a/libcxx/include/stdlib.h
+++ b/libcxx/include/stdlib.h
@@ -106,23 +106,8 @@ extern "C++" {
 #        undef llabs
 #      endif
 
-// MSVCRT already has the correct prototype in <stdlib.h> if __cplusplus is defined
-#      if !defined(_LIBCPP_MSVCRT)
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long abs(long __x) _NOEXCEPT { return __builtin_labs(__x); }
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long long abs(long long __x) _NOEXCEPT { return __builtin_llabs(__x); }
-#      endif // !defined(_LIBCPP_MSVCRT)
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI float abs(float __lcpp_x) _NOEXCEPT {
-  return __builtin_fabsf(__lcpp_x); // Use builtins to prevent needing math.h
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI double abs(double __lcpp_x) _NOEXCEPT {
-  return __builtin_fabs(__lcpp_x);
-}
-
-[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI long double abs(long double __lcpp_x) _NOEXCEPT {
-  return __builtin_fabsl(__lcpp_x);
-}
+#      include <__math/abs.h>
+using std::__math::abs;
 
 // div
 
diff --git a/libcxx/test/std/numerics/c.math/abs.verify.cpp b/libcxx/test/std/numerics/c.math/abs.verify.cpp
index dec80762b214..cc30112f0c31 100644
--- a/libcxx/test/std/numerics/c.math/abs.verify.cpp
+++ b/libcxx/test/std/numerics/c.math/abs.verify.cpp
@@ -13,10 +13,10 @@ void f() {
     (void)std::abs(ui); // expected-error {{call to 'abs' is ambiguous}}
 
     unsigned char uc = -5;
-    (void)std::abs(uc); // expected-warning {{taking the absolute value of unsigned type 'unsigned char' has no effect}}
+    (void)std::abs(uc); // expected-warning 0-1 {{taking the absolute value of unsigned type 'unsigned char' has no effect}}
 
     unsigned short us = -5;
-    (void)std::abs(us); // expected-warning {{taking the absolute value of unsigned type 'unsigned short' has no effect}}
+    (void)std::abs(us); // expected-warning 0-1 {{taking the absolute value of unsigned type 'unsigned short' has no effect}}
 
     unsigned long ul = -5;
     (void)std::abs(ul); // expected-error {{call to 'abs' is ambiguous}}
-- 
cgit v1.2.3


From 20245bbf66977ca9de5a2b6e29e8617a3a5d9fb5 Mon Sep 17 00:00:00 2001
From: Nikolas Klauser <nikolasklauser@berlin.de>
Date: Thu, 19 Jun 2025 10:41:10 +0200
Subject: [libc++][NFC] Format abs.pass.cpp test

I will modify the test in an upcoming PR. I'm formatting it now to avoid
a bunch of whitespace changes in that PR.
---
 libcxx/test/std/numerics/c.math/abs.pass.cpp | 80 +++++++++++++---------------
 1 file changed, 37 insertions(+), 43 deletions(-)

diff --git a/libcxx/test/std/numerics/c.math/abs.pass.cpp b/libcxx/test/std/numerics/c.math/abs.pass.cpp
index 03aae465c257..51aee6e98683 100644
--- a/libcxx/test/std/numerics/c.math/abs.pass.cpp
+++ b/libcxx/test/std/numerics/c.math/abs.pass.cpp
@@ -14,65 +14,59 @@
 
 #include "test_macros.h"
 
-template<class T>
-struct correct_size_int
-{
-    typedef typename std::conditional<sizeof(T) < sizeof(int), int, T>::type type;
+template <class T>
+struct correct_size_int {
+  typedef typename std::conditional<sizeof(T) < sizeof(int), int, T>::type type;
 };
 
 template <class Source, class Result>
-void test_abs()
-{
-    Source neg_val = -5;
-    Source pos_val = 5;
-    Result res = 5;
+void test_abs() {
+  Source neg_val = -5;
+  Source pos_val = 5;
+  Result res     = 5;
 
-    ASSERT_SAME_TYPE(decltype(std::abs(neg_val)), Result);
+  ASSERT_SAME_TYPE(decltype(std::abs(neg_val)), Result);
 
-    assert(std::abs(neg_val) == res);
-    assert(std::abs(pos_val) == res);
+  assert(std::abs(neg_val) == res);
+  assert(std::abs(pos_val) == res);
 }
 
-void test_big()
-{
-    long long int big_value = std::numeric_limits<long long int>::max(); // a value too big for ints to store
-    long long int negative_big_value = -big_value;
-    assert(std::abs(negative_big_value) == big_value); // make sure it doesn't get casted to a smaller type
+void test_big() {
+  long long int big_value          = std::numeric_limits<long long int>::max(); // a value too big for ints to store
+  long long int negative_big_value = -big_value;
+  assert(std::abs(negative_big_value) == big_value); // make sure it doesn't get casted to a smaller type
 }
 
 // The following is helpful to keep in mind:
 // 1byte == char <= short <= int <= long <= long long
 
-int main(int, char**)
-{
-    // On some systems char is unsigned.
-    // If that is the case, we should just test signed char twice.
-    typedef std::conditional<
-        std::is_signed<char>::value, char, signed char
-    >::type SignedChar;
+int main(int, char**) {
+  // On some systems char is unsigned.
+  // If that is the case, we should just test signed char twice.
+  typedef std::conditional< std::is_signed<char>::value, char, signed char >::type SignedChar;
 
-    // All types less than or equal to and not greater than int are promoted to int.
-    test_abs<short int, int>();
-    test_abs<SignedChar, int>();
-    test_abs<signed char, int>();
+  // All types less than or equal to and not greater than int are promoted to int.
+  test_abs<short int, int>();
+  test_abs<SignedChar, int>();
+  test_abs<signed char, int>();
 
-    // These three calls have specific overloads:
-    test_abs<int, int>();
-    test_abs<long int, long int>();
-    test_abs<long long int, long long int>();
+  // These three calls have specific overloads:
+  test_abs<int, int>();
+  test_abs<long int, long int>();
+  test_abs<long long int, long long int>();
 
-    // Here there is no guarantee that int is larger than int8_t so we
-    // use a helper type trait to conditional test against int.
-    test_abs<std::int8_t, correct_size_int<std::int8_t>::type>();
-    test_abs<std::int16_t, correct_size_int<std::int16_t>::type>();
-    test_abs<std::int32_t, correct_size_int<std::int32_t>::type>();
-    test_abs<std::int64_t, correct_size_int<std::int64_t>::type>();
+  // Here there is no guarantee that int is larger than int8_t so we
+  // use a helper type trait to conditional test against int.
+  test_abs<std::int8_t, correct_size_int<std::int8_t>::type>();
+  test_abs<std::int16_t, correct_size_int<std::int16_t>::type>();
+  test_abs<std::int32_t, correct_size_int<std::int32_t>::type>();
+  test_abs<std::int64_t, correct_size_int<std::int64_t>::type>();
 
-    test_abs<long double, long double>();
-    test_abs<double, double>();
-    test_abs<float, float>();
+  test_abs<long double, long double>();
+  test_abs<double, double>();
+  test_abs<float, float>();
 
-    test_big();
+  test_big();
 
-    return 0;
+  return 0;
 }
-- 
cgit v1.2.3


From 6273c5d4d3540204cb0d298cf1cf74ba94ed2a6c Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Thu, 19 Jun 2025 09:57:07 +0100
Subject: [lldb][AArch64] Correctly invalidate svg when vg is written (#140875)

Recently the Linux Kernel has fixed a bunch of issues in SME support and
while testing that, I found two tests failing:
FAIL: test_za_register_dynamic_config_main_disabled
(TestZAThreadedDynamic.AArch64ZAThreadedTestCase)
FAIL: test_za_register_dynamic_config_main_enabled
(TestZAThreadedDynamic.AArch64ZAThreadedTestCase)

These tests write to vg during streaming mode from lldb and expect to
see that za has been resized to match it. Instead, it was unavailable.
lldb-server was sending the correct amount of data but lldb client was
expecting the old size.

Turns out that instead of a write to vg invalidating svg, it was
invalidating... something else. I'm still not sure how these tests ever
worked but with this one line fix, they pass again.

I did not see this issue with SVE or streaming SVE Z registers because
those always resize using the value of vg, and vg always has the value
we just wrote.

(remember that vg is the vector length of the **current** mode, not of
non-streaming mode, whereas svg is the vector length of streaming mode,
even if you are currently in non-streaming mode)
---
 lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
index c004c0f3c3cf..fbf128553fd5 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
@@ -425,8 +425,7 @@ void RegisterInfoPOSIX_arm64::AddRegSetSME(bool has_zt) {
   //
   // This must be added now, rather than when vg is defined because SME is a
   // dynamic set that may or may not be present.
-  static uint32_t vg_invalidates[] = {sme_regnum + 1 /*svg*/,
-                                      LLDB_INVALID_REGNUM};
+  static uint32_t vg_invalidates[] = {GetRegNumSMESVG(), LLDB_INVALID_REGNUM};
   m_dynamic_reg_infos[GetRegNumSVEVG()].invalidate_regs = vg_invalidates;
 }
 
-- 
cgit v1.2.3


From c0a9c908a697a150f797d0dff7f0bcd3782abed9 Mon Sep 17 00:00:00 2001
From: Durgadoss R <durgadossr@nvidia.com>
Date: Thu, 19 Jun 2025 14:45:43 +0530
Subject: [MLIR][NVVM-Docs] Fix rendering of a few tables in NVVM Docs
 (#144764)

This patch corrects the formatting of tables
in the tcgen05 ld/st and smem_descriptor Ops.

Signed-off-by: Durgadoss R <durgadossr@nvidia.com>
---
 mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
index 2dd7ac29cfed..418931b93126 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/NVVMOps.td
@@ -3455,6 +3455,7 @@ def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", []> {
     properties of multiplicand matrix in shared memory including its location
     in the shared memory of the current CTA.
 
+    ```
     +-----------+------+------------------------------------------------------+
     | Bit-field | Size | Description                                          |
     +-----------+------+------------------------------------------------------+
@@ -3477,6 +3478,7 @@ def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", []> {
     |           |      |   6: 32-Byte swizzling                               |
     |           |      |   (Values 3, 5 and 7 are invalid)                    |
     +-----------+------+------------------------------------------------------+    
+    ```
 
     Example:
     ```mlir
@@ -3578,7 +3580,8 @@ def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld", [NVVMRequiresSMa<[100, 101]>]> {
     elements from adjacent columns into a single 32-bit element during the load.
 
     The following table describes the size of the vector for various combinations
-    of `num` and `shape` attributes
+    of `num` and `shape` attributes:
+    ```
     |=====================================================================|
     | num/shape      |     16x32bx2/16x64b/32x32b |  16x128b   | 16x256b  |
     |=====================================================================|
@@ -3591,6 +3594,7 @@ def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld", [NVVMRequiresSMa<[100, 101]>]> {
     | x64            |          64                |    128     |    NA    |
     | x128           |          128               |    NA      |    NA    |
     |=====================================================================|
+    ```
 
     Example:
     ```mlir
@@ -3666,7 +3670,8 @@ def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st", [NVVMRequiresSMa<[100, 101]>]> {
     in the register into two 16-bit elements and store them in adjacent columns.
 
     The following table describes the size of the vector for various combinations
-    of `num` and `shape` attributes
+    of `num` and `shape` attributes:
+    ```
     |=====================================================================|
     | num/shape      |     16x32bx2/16x64b/32x32b |  16x128b   | 16x256b  |
     |=====================================================================|
@@ -3679,6 +3684,7 @@ def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st", [NVVMRequiresSMa<[100, 101]>]> {
     | x64            |          64                |    128     |    NA    |
     | x128           |          128               |    NA      |    NA    |
     |=====================================================================|
+    ```
 
     Example:
     ```mlir
-- 
cgit v1.2.3


From 97c1a2444574b32dd7a283c53be248c5dbbf62e9 Mon Sep 17 00:00:00 2001
From: Fabian Mora <fmora.dev@gmail.com>
Date: Thu, 19 Jun 2025 05:47:44 -0400
Subject: [mlir][linalg] Add option to pad dynamic dims to
 `linalg::rewriteAsPaddedOp` (#144354)

This patch makes the following changes:

- Add a `ValueRange typeDynDims` argument to
`linalg::makeComposedPadHighOp`, allowing to pad a tensor with dynamic
dimensions using `tensor::createPadHighOp`.

- Add a `DenseMap<std::pair<unsigned, unsigned>, OpFoldResult>
sizeToPadTo;` option to `LinalgPaddingOptions`. This option allows
setting the size to use when padding a dimension of an operand, allowing
to pad operands even in the case they don't have a constant upper
bounding box. If the value is not provided, then the constant upper
bound is used by default.

- Add a `use_prescribed_tensor_shapes` option to
`transform.structured.pad`. If set to true then `tensor.dim` will be
used as dimensions to compute the size of the padded dim instead of
computing the constant upper bound.

- This patch also changes the behavior for computing the padded shape
`linalg::rewriteAsPaddedOp`, by using the newly added options in
`LinalgPaddingOptions`.

- Finally it adds tests verifying the behavior.
---
 .../Linalg/TransformOps/LinalgTransformOps.td      |  10 +-
 .../mlir/Dialect/Linalg/Transforms/Transforms.h    |  17 +++
 mlir/include/mlir/Dialect/Linalg/Utils/Utils.h     |  17 ++-
 .../Linalg/TransformOps/LinalgTransformOps.cpp     |  36 ++++-
 mlir/lib/Dialect/Linalg/Transforms/Padding.cpp     | 163 ++++++++++++++++-----
 mlir/lib/Dialect/Linalg/Utils/Utils.cpp            |  24 ++-
 mlir/test/Dialect/Linalg/transform-op-pad.mlir     |  39 ++++-
 7 files changed, 246 insertions(+), 60 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 15ea5e7bf715..6f6df350f1ba 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -1134,7 +1134,8 @@ def PadOp : Op<Transform_Dialect, "structured.pad",
          DefaultValuedAttr<
           TypedArrayAttrBase<I64ArrayAttr, "array of arrays of i64">,
           "{}">:$transpose_paddings,
-         DefaultValuedAttr<StrAttr, "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copy_back_op);
+         DefaultValuedAttr<StrAttr, "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copy_back_op,
+         DefaultValuedAttr<UnitAttr, "false">:$use_prescribed_tensor_shapes);
   let results = (outs TransformHandleTypeInterface:$padded,
                       TransformHandleTypeInterface:$pad,
                       TransformHandleTypeInterface:$copy);
@@ -1142,6 +1143,7 @@ def PadOp : Op<Transform_Dialect, "structured.pad",
   let assemblyFormat = [{
     $target 
     (`pad_to_multiple_of` custom<DynamicIndexList>($pad_to_multiple_of, $static_pad_to_multiple_of)^)?
+    (`use_prescribed_tensor_shapes` $use_prescribed_tensor_shapes^)?
     attr-dict
     `:` functional-type(operands, results)
   }];
@@ -1159,13 +1161,15 @@ def PadOp : Op<Transform_Dialect, "structured.pad",
                    CArg<"ArrayRef<int64_t>", "{}">:$staticPadToMultipleOf,
                    CArg<"ArrayRef<int64_t>", "{}">:$nofoldFlags,
                    CArg<"ArrayRef<Attribute>", "{}">:$transposePaddings,
-                   CArg<"StringRef", "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copyBackOp)>,
+                   CArg<"StringRef", "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copyBackOp,
+                   CArg<"bool", "false">:$usePrescribedTensorShapes)>,
     OpBuilder<(ins "Value":$target,
                    "ArrayRef<int64_t>":$paddingDimensions,
                    "ArrayRef<OpFoldResult>":$mixedPadToMultipleOf,
                    CArg<"ArrayRef<int64_t>", "{}">:$nofoldFlags,
                    CArg<"ArrayRef<Attribute>", "{}">:$transposePaddings,
-                   CArg<"StringRef", "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copyBackOp)>
+                   CArg<"StringRef", "::mlir::bufferization::MaterializeInDestinationOp::getOperationName()">:$copyBackOp,
+                   CArg<"bool", "false">:$usePrescribedTensorShapes)>
   ];
 
   let extraClassDeclaration = [{
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 2eef0a06d0eb..147a2907f52e 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -295,6 +295,23 @@ struct LinalgPaddingOptions {
     padToMultipleOf.emplace(m.begin(), m.end());
     return *this;
   }
+  /// A mapping between an operand and shape dim, and a size for a padding
+  /// dimension. Each size is expected to be greater or equal than the
+  /// corresponding shape dim. If no value is provided then the constant upper
+  /// bound will be used.
+  DenseMap<std::pair<unsigned, unsigned>, OpFoldResult> sizeToPadTo;
+  LinalgPaddingOptions &setSizeToPadTo(unsigned operandIndex, unsigned dimIndex,
+                                       OpFoldResult size) {
+    assert(size && "expected non-null size");
+    sizeToPadTo[{operandIndex, dimIndex}] = size;
+    return *this;
+  }
+  /// Given the operand index and shape dim it returns the size to pad to.
+  OpFoldResult getSizeToPadTo(unsigned operandIndex, unsigned dimIndex) const {
+    return sizeToPadTo.lookup_or(
+        std::pair<unsigned, unsigned>(operandIndex, dimIndex), nullptr);
+  }
+
   /// A flag for every operand to mark the PadOp as nofold which enables
   /// packing for statically shaped operands.
   SmallVector<bool> nofoldFlags;
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index 80aa034d2199..fc151d02ceef 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -71,12 +71,14 @@ bool isParallelIterator(utils::IteratorType iteratorType);
 /// Check if iterator type  has "reduction" semantics.
 bool isReductionIterator(utils::IteratorType iteratorType);
 
-/// Create a tensor::PadOp that pads `source` to the size of the statically
-/// sized `type` whose static sizes are assumed to be greater than the dynamic
-/// `source` size. The padding introduces trailing `pad` values until the
-/// target size is met. If `source` is defined by one or more LinalgOps that
-/// have been padded with the same value and sizes, return their padded result
-/// instead of creating a tensor::PadOp.
+/// Create a tensor::PadOp that pads `source` to the shape of `type` whose sizes
+/// are assumed to be greater than the dynamic `source` size. If `typeDynDims`
+/// is specified, then it must contain the sizes of all the dynamic dimensions
+/// in order of appearance in `type`, otherwise the function will pad those
+/// values to `0`. The padding introduces trailing `pad` values until the target
+/// size is met. If `source` is defined by one or more LinalgOps that have been
+/// padded with the same  value and sizes, return their padded result instead of
+/// creating a tensor::PadOp.
 ///
 /// Example:
 /// ```
@@ -91,7 +93,8 @@ bool isReductionIterator(utils::IteratorType iteratorType);
 /// %4 = tensor.pad %3 low[0, 0] high[...] { tensor.yield %other_cst }
 /// ```
 Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
-                            Value source, Value pad, bool nofold);
+                            Value source, Value padding, bool nofold,
+                            ValueRange typeDynDims = std::nullopt);
 
 /// Returns GenericOp that copies an n-D memref. Unlike the current
 /// implementation of memref::CopyOp, this op can further tile, lower to loops
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index b2c28f5eed33..d78c8847f884 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -1907,7 +1907,8 @@ void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
                              ArrayRef<int64_t> padToMultipleOf,
                              ArrayRef<int64_t> nofoldFlags,
                              ArrayRef<Attribute> transposePaddings,
-                             StringRef copyBackOp) {
+                             StringRef copyBackOp,
+                             bool usePrescribedTensorShapes) {
   auto resultType = transform::AnyOpType::get(b.getContext());
   return build(/*builder=*/b,
                /*result=*/result,
@@ -1922,7 +1923,9 @@ void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
                     : b.getDenseI64ArrayAttr(padToMultipleOf)),
                /*nofoldFlags=*/b.getI64ArrayAttr(nofoldFlags),
                /*transposePaddings=*/b.getArrayAttr(transposePaddings),
-               /*copyBackOp=*/b.getStringAttr(copyBackOp));
+               /*copyBackOp=*/b.getStringAttr(copyBackOp),
+               /*usePrescribedTensorShapes=*/
+               usePrescribedTensorShapes ? b.getUnitAttr() : nullptr);
 }
 
 void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
@@ -1930,7 +1933,8 @@ void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
                              ArrayRef<OpFoldResult> mixedPadToMultipleOf,
                              ArrayRef<int64_t> nofoldFlags,
                              ArrayRef<Attribute> transposePaddings,
-                             StringRef copyBackOp) {
+                             StringRef copyBackOp,
+                             bool usePrescribedTensorShapes) {
   auto resultType = transform::AnyOpType::get(b.getContext());
   SmallVector<int64_t> staticPadToMultipleOf;
   SmallVector<Value> dynamicPadToMultipleOf;
@@ -1946,7 +1950,8 @@ void transform::PadOp::build(OpBuilder &b, OperationState &result, Value target,
                /*padToMultipleOf=*/staticPadToMultipleOf,
                /*nofoldFlags=*/b.getI64ArrayAttr(nofoldFlags),
                /*transposePaddings=*/b.getArrayAttr(transposePaddings),
-               /*copyBackOp=*/b.getStringAttr(copyBackOp));
+               /*copyBackOp=*/copyBackOp,
+               /*usePrescribedTensorShapes=*/usePrescribedTensorShapes);
 }
 
 void PadOp::getEffects(
@@ -2051,11 +2056,34 @@ transform::PadOp::apply(transform::TransformRewriter &rewriter,
     } else {
       llvm_unreachable("unsupported copy_back op");
     }
+    // Populate `sizeToPadTo` with the dynamic tensor sizes for each operand.
+    bool irChanged = false;
+    if (getUsePrescribedTensorShapes() &&
+        linalgTarget.hasPureTensorSemantics()) {
+      OpBuilder::InsertionGuard g(rewriter);
+      rewriter.setInsertionPoint(linalgTarget);
+      for (OpOperand &operand : linalgTarget->getOpOperands()) {
+        for (auto [i, dim] : llvm::enumerate(linalgTarget.getShape(&operand))) {
+          if (!ShapedType::isDynamic(dim))
+            continue;
+          options.setSizeToPadTo(operand.getOperandNumber(), i,
+                                 tensor::getMixedSize(rewriter,
+                                                      operand.get().getLoc(),
+                                                      operand.get(), i));
+          irChanged = true;
+        }
+      }
+    }
 
     SmallVector<Value> replacements;
     SmallVector<tensor::PadOp> newPadOps;
     if (failed(rewriteAsPaddedOp(rewriter, linalgTarget, options, paddedOp,
                                  replacements, newPadOps))) {
+      if (irChanged) {
+        auto diag = emitDefiniteFailure() << "failed to pad op";
+        diag.attachNote(target->getLoc()) << "target op";
+        return diag;
+      }
       auto diag = emitSilenceableError() << "failed to pad op";
       diag.attachNote(target->getLoc()) << "target op";
       return diag;
diff --git a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
index 9a685f6dc96a..dc9e11eccac4 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/Padding.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Linalg/Transforms/Transforms.h"
 
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
 #include "mlir/Dialect/Linalg/IR/Linalg.h"
@@ -22,53 +23,93 @@ using namespace mlir::linalg;
 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
 #define DBGSNL() (llvm::dbgs() << "\n")
 
-/// Compute the padded shape of the given operand. The operand is padded to a
-/// static bounding box according to the specified padding options.
-static LogicalResult computePaddedShape(linalg::LinalgOp opToPad,
+namespace {
+/// Helper class for storing padding information.
+struct PaddingInfo {
+  PaddingInfo(int64_t padToMultipleOf = 1, OpFoldResult size = {})
+      : padToMultipleOf(padToMultipleOf), size(size) {}
+  /// Pad the tensor to a multiple of.
+  int64_t padToMultipleOf = 1;
+  /// The size used for padding.
+  OpFoldResult size = {};
+};
+
+/// Helper class for storing and computing the padded shape.
+struct PaddedShape {
+  /// Initializes the shape information and on success it returns whether the
+  /// shape of the operand will change. Returns failure if the operand cannot be
+  /// padded.
+  FailureOr<bool> initialize(linalg::LinalgOp opToPad, OpOperand *opOperand,
+                             const LinalgPaddingOptions &options);
+
+  /// Computs the padded shape.
+  void computePadding(OpBuilder &builder, Value operand);
+
+  /// Returns the new tensor type.
+  RankedTensorType getType(Type elemTy) {
+    return RankedTensorType::get(shape, elemTy);
+  }
+
+  SmallVector<Value> dynDims;
+
+private:
+  SmallVector<int64_t> shape;
+  DenseMap<int64_t, PaddingInfo> dimToInfo;
+};
+} // namespace
+
+FailureOr<bool> PaddedShape::initialize(linalg::LinalgOp opToPad,
                                         OpOperand *opOperand,
-                                        const LinalgPaddingOptions &options,
-                                        SmallVector<int64_t> &paddedShape,
-                                        bool &alreadyHasRequestedShape) {
+                                        const LinalgPaddingOptions &options) {
   AffineMap indexingMap = opToPad.getMatchingIndexingMap(opOperand);
-  ArrayRef<int64_t> shape = opToPad.getShape(opOperand);
+
+  // Initialize the padded shape.
+  llvm::append_range(shape, opToPad.getShape(opOperand));
 
   // Collect the shape dimensions that are a function of "paddingDimensions",
   // along with the multiple that they should be padded to ("1" if none).
-  alreadyHasRequestedShape = true;
-  DenseMap<int64_t, int64_t> shapeDimToMultiple;
+  bool alreadyHasRequestedShape = true;
   for (const auto &dimEn : enumerate(options.paddingDimensions)) {
     for (const auto &en : enumerate(indexingMap.getResults())) {
       if (en.value().isFunctionOfDim(dimEn.value())) {
+        PaddingInfo paddingInfo;
         int64_t dimSize = shape[en.index()];
         if (options.padToMultipleOf.has_value()) {
-          shapeDimToMultiple[en.index()] =
+          paddingInfo.padToMultipleOf =
               (*options.padToMultipleOf)[dimEn.index()];
         } else {
-          shapeDimToMultiple[en.index()] = 1;
+          paddingInfo.padToMultipleOf = 1;
         }
-        if (ShapedType::isDynamic(dimSize)) {
-          alreadyHasRequestedShape = false;
-        } else if (dimSize % shapeDimToMultiple[en.index()] != 0) {
+
+        // Check if the user provided a size in the options.
+        paddingInfo.size =
+            options.getSizeToPadTo(opOperand->getOperandNumber(), en.index());
+
+        // Set the padding info.
+        dimToInfo[en.index()] = paddingInfo;
+        if (ShapedType::isDynamic(dimSize) ||
+            dimSize % paddingInfo.padToMultipleOf != 0 ||
+            !paddingInfo.size.isNull()) {
           alreadyHasRequestedShape = false;
         }
       }
     }
   }
 
-  // Helper function to round a number up to a given multiple.
-  auto ceil = [](int64_t val, int64_t multiple) {
-    return ((val + multiple - 1) / multiple) * multiple;
-  };
-
   // Upper bound the sizes to obtain a static bounding box.
-  paddedShape.assign(shape.begin(), shape.end());
   for (int64_t i = 0, e = shape.size(); i < e; ++i) {
-    LLVM_DEBUG(DBGS() << "--compute padded size for dim " << i << "\n");
+    LLVM_DEBUG(DBGS() << "--computing un-padded size for dim " << i << "\n");
     // Skip dimensions that do not require padding.
-    if (!shapeDimToMultiple.contains(i)) {
+    if (!dimToInfo.contains(i)) {
       LLVM_DEBUG(DBGS() << "----dim does not require padding, SKIP\n");
       continue;
     }
+    PaddingInfo &info = dimToInfo[i];
+    if (info.size) {
+      LLVM_DEBUG(DBGS() << "----the user provided the size: " << info.size
+                        << "\n");
+      continue;
+    }
     // Otherwise, try to compute a constant upper bound for the size value.
     FailureOr<int64_t> upperBound =
         ValueBoundsConstraintSet::computeConstantBound(
@@ -77,14 +118,58 @@ static LogicalResult computePaddedShape(linalg::LinalgOp opToPad,
              /*dim=*/i},
             /*stopCondition=*/nullptr, /*closedUB=*/true);
     if (failed(upperBound)) {
-      LLVM_DEBUG(DBGS() << "----could not compute a bounding box for padding");
+      LLVM_DEBUG(
+          DBGS() << "----could not compute a bounding box for padding\n");
       return failure();
     }
-    paddedShape[i] = ceil(*upperBound, shapeDimToMultiple[i]);
-    LLVM_DEBUG(DBGS() << "----new dim size: " << paddedShape[i] << "\n");
+    info.size =
+        IntegerAttr::get(IndexType::get(opToPad.getContext()), *upperBound);
+    LLVM_DEBUG(DBGS() << "----new un-padded size: " << info.size << "\n");
   }
+  return alreadyHasRequestedShape;
+}
 
-  return success();
+void PaddedShape::computePadding(OpBuilder &builder, Value operand) {
+  Location loc = operand.getLoc();
+  AffineExpr sizeSym = builder.getAffineSymbolExpr(0);
+
+  // Compute the padding for each dimension.
+  for (auto &&[i, dim] : llvm::enumerate(shape)) {
+    LLVM_DEBUG(DBGS() << "--computing padded size for dim " << i << "\n");
+
+    // Get the padding info or default info for the shape dimension.
+    PaddingInfo paddingInfo = dimToInfo.lookup(i);
+
+    // Skip dimensions that do not require padding.
+    if (paddingInfo.size.isNull()) {
+      LLVM_DEBUG(DBGS() << "----dim does not require padding, SKIP\n");
+
+      // We still need to push the size as `makeComposedPadHighOp` expects a
+      // range with all the dynamic sizes, whether they're being padded or not.
+      if (ShapedType::isDynamic(dim)) {
+        dynDims.push_back(
+            cast<Value>(tensor::getMixedSize(builder, loc, operand, i)));
+      }
+      continue;
+    }
+
+    // Compute the padded size to be a multiple of `padToMultipleOf`.
+    AffineExpr szExpr = (sizeSym).ceilDiv(paddingInfo.padToMultipleOf) *
+                        paddingInfo.padToMultipleOf;
+    OpFoldResult paddedSize = affine::makeComposedFoldedAffineApply(
+        builder, loc, szExpr, paddingInfo.size);
+    assert(paddedSize && "invalid arguments to affine apply");
+
+    if (auto cstSzAttr = dyn_cast<Attribute>(paddedSize)) {
+      // Update the shape as the size is static.
+      dim = cast<IntegerAttr>(cstSzAttr).getValue().getZExtValue();
+    } else {
+      // Add a dynamic dimension.
+      dim = ShapedType::kDynamic;
+      dynDims.push_back(cast<Value>(paddedSize));
+    }
+    LLVM_DEBUG(DBGS() << "----new dim size: " << paddedSize << "\n");
+  }
 }
 
 /// Pad the `opOperand` in the "paddingDimensions" using the padding value and
@@ -107,20 +192,21 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
        options.padToMultipleOf->size() == options.paddingDimensions.size()) &&
       "invalid number of elements in padToMultipleOf");
 
-  // Compute padded shape.
-  SmallVector<int64_t> paddedShape;
-  bool alreadyHasRequestedShape = false;
-  if (failed(computePaddedShape(opToPad, opOperand, options, paddedShape,
-                                alreadyHasRequestedShape)))
+  // Initialize the padded shape and get whether it requires padding.
+  PaddedShape shape;
+  FailureOr<bool> alreadyHasRequestedShape =
+      shape.initialize(opToPad, opOperand, options);
+  if (failed(alreadyHasRequestedShape)) {
     return rewriter.notifyMatchFailure(opToPad,
                                        "--failed to compute padded shape");
+  }
 
-  // Return the unpadded operand if padding to a static shape is not needed and
+  // Return the un-padded operand if padding to a static shape is not needed and
   // if the nofold flag is not set.
   bool nofold = opOperand->getOperandNumber() < options.nofoldFlags.size()
                     ? bool(options.nofoldFlags[opOperand->getOperandNumber()])
                     : false;
-  if (!nofold && alreadyHasRequestedShape)
+  if (!nofold && *alreadyHasRequestedShape)
     return opOperand->get();
 
   // Fail if `paddingValues` specifies no padding value.
@@ -140,13 +226,18 @@ static FailureOr<Value> padOperandToSmallestStaticBoundingBox(
         opToPad.getLoc(), cast<TypedAttr>(paddingAttr));
   }
 
+  // Computes the padded shape.
+  if (!*alreadyHasRequestedShape)
+    shape.computePadding(rewriter, opOperand->get());
+
   // Pad the operand to the bounding box defined by `paddedShape`.
-  auto paddedTensorType = RankedTensorType::get(
-      paddedShape, getElementTypeOrSelf(opOperand->get()));
+  RankedTensorType paddedTensorType =
+      shape.getType(getElementTypeOrSelf(opOperand->get()));
   LLVM_DEBUG(DBGS() << "--SUCCESS, makeComposedPadHighOp with type: "
                     << paddedTensorType);
   return makeComposedPadHighOp(rewriter, opToPad->getLoc(), paddedTensorType,
-                               opOperand->get(), paddingValue, nofold);
+                               opOperand->get(), paddingValue, nofold,
+                               shape.dynDims);
 }
 
 LogicalResult
diff --git a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
index 2527d90cfa2e..209309ddb413 100644
--- a/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
+++ b/mlir/lib/Dialect/Linalg/Utils/Utils.cpp
@@ -244,11 +244,13 @@ bool isReductionIterator(utils::IteratorType iteratorType) {
 }
 
 Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
-                            Value source, Value pad, bool nofold) {
+                            Value source, Value pad, bool nofold,
+                            ValueRange typeDynDims) {
   // Exit if `source` is not defined by an ExtractSliceOp.
   auto sliceOp = source.getDefiningOp<tensor::ExtractSliceOp>();
   if (!sliceOp)
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Search the `source` use-def chain for padded LinalgOps.
   Value current = sliceOp.getSource();
@@ -264,24 +266,28 @@ Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
   // Exit if the search fails to match a tensor::PadOp at the end of the matched
   // LinalgOp sequence.
   if (!padOp)
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Exit if the padded result type does not match.
   if (sliceOp.getSource().getType() != type)
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Exit if the LinalgOps are not high padded.
   if (llvm::any_of(padOp.getMixedLowPad(), [](OpFoldResult ofr) {
         return getConstantIntValue(ofr) != static_cast<int64_t>(0);
       }))
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Exit if `padOpSliceOp`, which defines the slice used by
   // `padOp`, is rank-reducing.
   auto padOpSliceOp = padOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
   if (!padOpSliceOp ||
       sliceOp.getMixedSizes().size() != padOpSliceOp.getMixedSizes().size())
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Exit if the sizes of the dynamic sizes of `sliceOp` do not match the size
   // of the slice padded by `padOp`.
@@ -290,14 +296,16 @@ Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
           [](std::tuple<OpFoldResult, OpFoldResult> it) {
             return !isEqualConstantIntOrValue(std::get<0>(it), std::get<1>(it));
           }))
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Exit if the padding values do not match.
   Attribute padOpPadAttr, padAttr;
   Value padOpPad = padOp.getConstantPaddingValue();
   if (!padOpPad || !matchPattern(padOpPad, m_Constant(&padOpPadAttr)) ||
       !matchPattern(pad, m_Constant(&padAttr)) || padOpPadAttr != padAttr)
-    return tensor::createPadHighOp(type, source, pad, nofold, loc, b);
+    return tensor::createPadHighOp(type, source, pad, nofold, loc, b,
+                                   typeDynDims);
 
   // Return the padded result if the padding values and sizes match.
   return sliceOp.getSource();
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad.mlir b/mlir/test/Dialect/Linalg/transform-op-pad.mlir
index ab2711545405..bc684b53c9b6 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad.mlir
@@ -300,7 +300,7 @@ func.func @negative_no_ub_estimate(%arg0: tensor<?x12xf32>,
 module attributes {transform.with_named_sequence} {
   transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
     %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
-    // expected-error @below {{ailed to pad op}}
+    // expected-error @below {{failed to pad op}}
     %padded, %pad, %copy_back = transform.structured.pad %0 {
       padding_values=[0.0 : f32, 0.0 : f32, 0.0 : f32],
       // Note - attempting to pad non-static dim
@@ -313,6 +313,41 @@ module attributes {transform.with_named_sequence} {
 
 // -----
 
+// Test dynamic padding using `use_prescribed_tensor_shapes`
+
+// CHECK: #[[MAP:.*]] = affine_map<()[s0] -> (-s0 + (s0 ceildiv 7) * 7)>
+// CHECK: @use_prescribed_tensor_shapes
+// CHECK: (%[[ARG0:.*]]: tensor<?x12xf32>, %[[ARG1:.*]]: tensor<12x?xf32>
+func.func @use_prescribed_tensor_shapes(%arg0: tensor<?x12xf32>,
+                                   %arg1: tensor<12x?xf32>,
+                                   %arg2: tensor<?x?xf32>) -> tensor<?x?xf32> {
+  // CHECK: %[[C1_0:.*]] = arith.constant 1 : index
+  // CHECK: %[[DIM_0:.*]] = tensor.dim %[[ARG1]], %[[C1_0]] : tensor<12x?xf32>
+  // CHECK: %[[PADDING:.*]] = affine.apply #[[MAP]]()[%[[DIM_0]]]
+  // CHECK: %[[PADDED:.*]] = tensor.pad %[[ARG1]] low[0, 0] high[0, %[[PADDING]]] {
+  // CHECK: linalg.matmul ins(%[[ARG0]], %[[PADDED]] : tensor<?x12xf32>, tensor<12x?xf32>) 
+  %0 = linalg.matmul ins(%arg0, %arg1 : tensor<?x12xf32>, tensor<12x?xf32>) outs(%arg2 : tensor<?x?xf32>) -> tensor<?x?xf32>
+  func.return %0 : tensor<?x?xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %0 = transform.structured.match ops{["linalg.matmul"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    %padded, %pad, %copy_back = transform.structured.pad %0
+    pad_to_multiple_of [7] use_prescribed_tensor_shapes {
+      padding_values=[0.0 : f32, 0.0 : f32, 0.0 : f32],
+      padding_dimensions=[1]
+    } : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
+    %func = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+    transform.apply_patterns to %func {
+      transform.apply_patterns.canonicalization
+    } {apply_cse} : !transform.any_op
+    transform.yield
+  }
+}
+
+// -----
+
 // Check that the padding can be applied even when the output argument of the
 // linalg op is not produced by an empty op or an extract_slice op.
 
@@ -416,6 +451,6 @@ module attributes {transform.with_named_sequence} {
       padding_dimensions=[0, 1, 2],
       nofold_flags=[1, 1, 1]
     } : (!transform.any_op) -> (!transform.any_op, !transform.any_op, !transform.any_op)
-    transform.yield
+    transform.yield 
   }
 }
-- 
cgit v1.2.3


From af51c9d9df9d482503fe30c80dd788a02161cea6 Mon Sep 17 00:00:00 2001
From: David Sherwood <david.sherwood@arm.com>
Date: Thu, 19 Jun 2025 10:49:40 +0100
Subject: [LV][NFC] Add branch weight test showing incorrect behaviour
 (#144682)

This patch adds a test that shows incorrect branch weights being set in
function

EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    |   2 +
 .../Transforms/LoopVectorize/branch-weights.ll     | 182 ++++++++++++++-------
 2 files changed, 129 insertions(+), 55 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 3b16248f962b..e14f985efd96 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7683,6 +7683,8 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
   BranchInst &BI =
       *BranchInst::Create(Bypass, LoopVectorPreHeader, CheckMinIters);
   if (hasBranchWeightMD(*OrigLoop->getLoopLatch()->getTerminator())) {
+    // FIXME: See test Transforms/LoopVectorize/branch-weights.ll. I don't
+    // think the MainLoopStep is correct.
     unsigned MainLoopStep = UF * VF.getKnownMinValue();
     unsigned EpilogueLoopStep =
         EPI.EpilogueUF * EPI.EpilogueVF.getKnownMinValue();
diff --git a/llvm/test/Transforms/LoopVectorize/branch-weights.ll b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
index e11f77d8aeae..6892709f085f 100644
--- a/llvm/test/Transforms/LoopVectorize/branch-weights.ll
+++ b/llvm/test/Transforms/LoopVectorize/branch-weights.ll
@@ -1,53 +1,103 @@
-; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4  -enable-epilogue-vectorization -epilogue-vectorization-force-VF=4 | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "br " --filter "^.*:" --filter "icmp" --version 5
+; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4  -enable-epilogue-vectorization \
+; RUN:   -epilogue-vectorization-force-VF=4 | FileCheck %s --check-prefix=MAINVF4IC1_EPI4
+; RUN: opt < %s -S -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4  -enable-epilogue-vectorization \
+; RUN:   -epilogue-vectorization-force-VF=4 | FileCheck %s --check-prefix=MAINVF4IC2_EPI4
 
-; CHECK-LABEL: @f0(
-;
-; CHECK: entry:
-; CHECK:   br i1 %cmp.entry, label %iter.check, label %exit, !prof [[PROF_F0_ENTRY:![0-9]+]]
-;
-; CHECK: iter.check:
-; CHECK:   br i1 %min.iters.check, label %vec.epilog.scalar.ph, label %vector.scevcheck, !prof [[PROF_F0_UNLIKELY:![0-9]+]]
-;
-; CHECK: vector.scevcheck:
-; CHECK:   br i1 %4, label %vec.epilog.scalar.ph, label %vector.main.loop.iter.check, !prof [[PROF_F0_UNLIKELY]]
-;
-; CHECK: vector.main.loop.iter.check:
-; CHECK:   br i1 %min.iters.check1, label %vec.epilog.ph, label %vector.ph, !prof [[PROF_F0_UNLIKELY]]
-;
-; CHECK: vector.ph:
-; CHECK:   br label %vector.body
-;
-; CHECK: vector.body:
-; CHECK:   br i1 {{.+}}, label %middle.block, label %vector.body, !prof [[PROF_F0_VECTOR_BODY:![0-9]+]]
-;
-; CHECK: middle.block:
-; CHECK:   br i1 %cmp.n, label %exit.loopexit, label %vec.epilog.iter.check, !prof [[PROF_F0_MIDDLE_BLOCKS:![0-9]+]]
-;
-; CHECK: vec.epilog.iter.check:
-; CHECK:   br i1 %min.epilog.iters.check, label %vec.epilog.scalar.ph, label %vec.epilog.ph, !prof [[PROF_F0_VEC_EPILOGUE_SKIP:![0-9]+]]
-;
-; CHECK: vec.epilog.ph:
-; CHECK:   br label %vec.epilog.vector.body
-;
-; CHECK: vec.epilog.vector.body:
-; CHECK:   br i1 {{.+}}, label %vec.epilog.middle.block, label %vec.epilog.vector.body, !prof [[PROF_F0_VEC_EPILOG_VECTOR_BODY:![0-9]+]]
-;
-; CHECK: vec.epilog.middle.block:
-; CHECK:   br i1 %cmp.n{{.+}}, label %exit.loopexit, label %vec.epilog.scalar.ph, !prof [[PROF_F0_MIDDLE_BLOCKS:![0-9]+]]
-;
-; CHECK: vec.epilog.scalar.ph:
-; CHECK:   br label %loop
-;
-; CHECK: loop:
-; CHECK:   br i1 %cmp.loop, label %loop, label %exit.loopexit, !prof [[PROF_F0_LOOP:![0-9]+]]
+; FIXME: For MAINVF4IC2_EPI4 the branch weights in the terminator of
+; the VEC_EPILOG_ITER_CHECK block should be [4,4] since we process 8
+; scalar iterations in the main loop, leaving the remaining count to
+; be in the range [0,7]. That gives a 4:4 chance of skipping the
+; vector epilogue. I believe the problem lies in
+; EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck
+; where the main loop VF is set to the same value as the epilogue VF.
+define void @f0(i8 %n, i32 %len, ptr %p) !prof !0 {
+; MAINVF4IC1_EPI4-LABEL: define void @f0(
+; MAINVF4IC1_EPI4-SAME: i8 [[N:%.*]], i32 [[LEN:%.*]], ptr [[P:%.*]]) !prof [[PROF0:![0-9]+]] {
+; MAINVF4IC1_EPI4:  [[ENTRY:.*:]]
+; MAINVF4IC1_EPI4:    [[CMP_ENTRY:%.*]] = icmp sgt i32 [[LEN]], 0
+; MAINVF4IC1_EPI4:    br i1 [[CMP_ENTRY]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]], !prof [[PROF1:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[ITER_CHECK]]:
+; MAINVF4IC1_EPI4:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0:%.*]], 4
+; MAINVF4IC1_EPI4:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]], !prof [[PROF2:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[VECTOR_SCEVCHECK]]:
+; MAINVF4IC1_EPI4:    [[TMP2:%.*]] = icmp slt i8 [[TMP1:%.*]], 0
+; MAINVF4IC1_EPI4:    [[TMP3:%.*]] = icmp ugt i32 [[LEN]], 255
+; MAINVF4IC1_EPI4:    br i1 [[TMP4:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC1_EPI4:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[TMP0]], 4
+; MAINVF4IC1_EPI4:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC1_EPI4:  [[VECTOR_PH]]:
+; MAINVF4IC1_EPI4:    br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4:  [[VECTOR_BODY]]:
+; MAINVF4IC1_EPI4:    [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC1_EPI4:    br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF3:![0-9]+]], !llvm.loop [[LOOP4:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; MAINVF4IC1_EPI4:    br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF7:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC1_EPI4:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_VEC_REMAINING:%.*]], 4
+; MAINVF4IC1_EPI4:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[VEC_EPILOG_PH]]:
+; MAINVF4IC1_EPI4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC1_EPI4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC1_EPI4:    [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT6:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC1_EPI4:    br i1 [[TMP12]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC1_EPI4:    [[CMP_N8:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC3]]
+; MAINVF4IC1_EPI4:    br i1 [[CMP_N8]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF7]]
+; MAINVF4IC1_EPI4:  [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC1_EPI4:    br label %[[LOOP:.*]]
+; MAINVF4IC1_EPI4:  [[LOOP]]:
+; MAINVF4IC1_EPI4:    [[CMP_LOOP:%.*]] = icmp ult i32 [[I32:%.*]], [[LEN]]
+; MAINVF4IC1_EPI4:    br i1 [[CMP_LOOP]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !prof [[PROF11:![0-9]+]], !llvm.loop [[LOOP12:![0-9]+]]
+; MAINVF4IC1_EPI4:  [[EXIT_LOOPEXIT]]:
+; MAINVF4IC1_EPI4:    br label %[[EXIT]]
+; MAINVF4IC1_EPI4:  [[EXIT]]:
 ;
-; CHECK: exit.loopexit:
-; CHECK:   br label %exit
+; MAINVF4IC2_EPI4-LABEL: define void @f0(
+; MAINVF4IC2_EPI4-SAME: i8 [[N:%.*]], i32 [[LEN:%.*]], ptr [[P:%.*]]) !prof [[PROF0:![0-9]+]] {
+; MAINVF4IC2_EPI4:  [[ENTRY:.*:]]
+; MAINVF4IC2_EPI4:    [[CMP_ENTRY:%.*]] = icmp sgt i32 [[LEN]], 0
+; MAINVF4IC2_EPI4:    br i1 [[CMP_ENTRY]], label %[[ITER_CHECK:.*]], label %[[EXIT:.*]], !prof [[PROF1:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[ITER_CHECK]]:
+; MAINVF4IC2_EPI4:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP0:%.*]], 4
+; MAINVF4IC2_EPI4:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]], !prof [[PROF2:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[VECTOR_SCEVCHECK]]:
+; MAINVF4IC2_EPI4:    [[TMP2:%.*]] = icmp slt i8 [[TMP1:%.*]], 0
+; MAINVF4IC2_EPI4:    [[TMP3:%.*]] = icmp ugt i32 [[LEN]], 255
+; MAINVF4IC2_EPI4:    br i1 [[TMP4:%.*]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4:  [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
+; MAINVF4IC2_EPI4:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i32 [[TMP0]], 8
+; MAINVF4IC2_EPI4:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]], !prof [[PROF2]]
+; MAINVF4IC2_EPI4:  [[VECTOR_PH]]:
+; MAINVF4IC2_EPI4:    br label %[[VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4:  [[VECTOR_BODY]]:
+; MAINVF4IC2_EPI4:    [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT:%.*]], [[N_VEC:%.*]]
+; MAINVF4IC2_EPI4:    br i1 [[TMP9]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !prof [[PROF3:![0-9]+]], !llvm.loop [[LOOP4:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4:    [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]]
+; MAINVF4IC2_EPI4:    br i1 [[CMP_N]], label %[[EXIT_LOOPEXIT:.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]], !prof [[PROF7:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[VEC_EPILOG_ITER_CHECK]]:
+; MAINVF4IC2_EPI4:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i32 [[N_VEC_REMAINING:%.*]], 4
+; MAINVF4IC2_EPI4:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF8:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[VEC_EPILOG_PH]]:
+; MAINVF4IC2_EPI4:    br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
+; MAINVF4IC2_EPI4:  [[VEC_EPILOG_VECTOR_BODY]]:
+; MAINVF4IC2_EPI4:    [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT6:%.*]], [[N_VEC3:%.*]]
+; MAINVF4IC2_EPI4:    br i1 [[TMP13]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !prof [[PROF9:![0-9]+]], !llvm.loop [[LOOP10:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[VEC_EPILOG_MIDDLE_BLOCK]]:
+; MAINVF4IC2_EPI4:    [[CMP_N8:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC3]]
+; MAINVF4IC2_EPI4:    br i1 [[CMP_N8]], label %[[EXIT_LOOPEXIT]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF11:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[VEC_EPILOG_SCALAR_PH]]:
+; MAINVF4IC2_EPI4:    br label %[[LOOP:.*]]
+; MAINVF4IC2_EPI4:  [[LOOP]]:
+; MAINVF4IC2_EPI4:    [[CMP_LOOP:%.*]] = icmp ult i32 [[I32:%.*]], [[LEN]]
+; MAINVF4IC2_EPI4:    br i1 [[CMP_LOOP]], label %[[LOOP]], label %[[EXIT_LOOPEXIT]], !prof [[PROF12:![0-9]+]], !llvm.loop [[LOOP13:![0-9]+]]
+; MAINVF4IC2_EPI4:  [[EXIT_LOOPEXIT]]:
+; MAINVF4IC2_EPI4:    br label %[[EXIT]]
+; MAINVF4IC2_EPI4:  [[EXIT]]:
 ;
-; CHECK: exit:
-; CHECK:   ret void
-
-define void @f0(i8 %n, i32 %len, ptr %p) !prof !0 {
 entry:
   %cmp.entry = icmp sgt i32 %len, 0
   br i1 %cmp.entry, label %loop, label %exit, !prof !1
@@ -72,11 +122,33 @@ exit:
 !0 = !{!"function_entry_count", i64 13}
 !1 = !{!"branch_weights", i32 12, i32 1}
 !2 = !{!"branch_weights", i32 1234, i32 1}
-
-; CHECK: [[PROF_F0_ENTRY]] = !{!"branch_weights", i32 12, i32 1}
-; CHECK: [[PROF_F0_UNLIKELY]] = !{!"branch_weights", i32 1, i32 127}
-; CHECK: [[PROF_F0_VECTOR_BODY]] = !{!"branch_weights", i32 1, i32 307}
-; CHECK: [[PROF_F0_MIDDLE_BLOCKS]] =  !{!"branch_weights", i32 1, i32 3}
-; CHECK: [[PROF_F0_VEC_EPILOGUE_SKIP]] = !{!"branch_weights", i32 4, i32 0}
-; CHECK: [[PROF_F0_VEC_EPILOG_VECTOR_BODY]] = !{!"branch_weights", i32 0, i32 0}
-; CHECK: [[PROF_F0_LOOP]] = !{!"branch_weights", i32 2, i32 1}
+;.
+; MAINVF4IC1_EPI4: [[PROF0]] = !{!"function_entry_count", i64 13}
+; MAINVF4IC1_EPI4: [[PROF1]] = !{!"branch_weights", i32 12, i32 1}
+; MAINVF4IC1_EPI4: [[PROF2]] = !{!"branch_weights", i32 1, i32 127}
+; MAINVF4IC1_EPI4: [[PROF3]] = !{!"branch_weights", i32 1, i32 307}
+; MAINVF4IC1_EPI4: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; MAINVF4IC1_EPI4: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAINVF4IC1_EPI4: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAINVF4IC1_EPI4: [[PROF7]] = !{!"branch_weights", i32 1, i32 3}
+; MAINVF4IC1_EPI4: [[PROF8]] = !{!"branch_weights", i32 4, i32 0}
+; MAINVF4IC1_EPI4: [[PROF9]] = !{!"branch_weights", i32 0, i32 0}
+; MAINVF4IC1_EPI4: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]], [[META6]]}
+; MAINVF4IC1_EPI4: [[PROF11]] = !{!"branch_weights", i32 2, i32 1}
+; MAINVF4IC1_EPI4: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
+;.
+; MAINVF4IC2_EPI4: [[PROF0]] = !{!"function_entry_count", i64 13}
+; MAINVF4IC2_EPI4: [[PROF1]] = !{!"branch_weights", i32 12, i32 1}
+; MAINVF4IC2_EPI4: [[PROF2]] = !{!"branch_weights", i32 1, i32 127}
+; MAINVF4IC2_EPI4: [[PROF3]] = !{!"branch_weights", i32 1, i32 153}
+; MAINVF4IC2_EPI4: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]], [[META6:![0-9]+]]}
+; MAINVF4IC2_EPI4: [[META5]] = !{!"llvm.loop.isvectorized", i32 1}
+; MAINVF4IC2_EPI4: [[META6]] = !{!"llvm.loop.unroll.runtime.disable"}
+; MAINVF4IC2_EPI4: [[PROF7]] = !{!"branch_weights", i32 1, i32 7}
+; MAINVF4IC2_EPI4: [[PROF8]] = !{!"branch_weights", i32 4, i32 0}
+; MAINVF4IC2_EPI4: [[PROF9]] = !{!"branch_weights", i32 0, i32 0}
+; MAINVF4IC2_EPI4: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]], [[META6]]}
+; MAINVF4IC2_EPI4: [[PROF11]] = !{!"branch_weights", i32 1, i32 3}
+; MAINVF4IC2_EPI4: [[PROF12]] = !{!"branch_weights", i32 2, i32 1}
+; MAINVF4IC2_EPI4: [[LOOP13]] = distinct !{[[LOOP13]], [[META5]]}
+;.
-- 
cgit v1.2.3


From 0fe78c4a290517925acc03d59f235926f440f155 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Wed, 18 Jun 2025 13:44:46 +0800
Subject: [NFC] [Serialization] Some Code Cleanups for Name lookup table things

---
 clang/include/clang/Serialization/ASTReader.h      |  26 ++-
 .../include/clang/Serialization/ASTRecordWriter.h  |   7 +
 clang/include/clang/Serialization/ASTWriter.h      |  10 +-
 clang/lib/Serialization/ASTReader.cpp              |  30 ++--
 clang/lib/Serialization/ASTReaderDecl.cpp          |  71 ++++----
 clang/lib/Serialization/ASTWriter.cpp              | 191 ++++++++++++---------
 clang/lib/Serialization/ASTWriterDecl.cpp          |  17 +-
 7 files changed, 186 insertions(+), 166 deletions(-)

diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index ba676fd8698e..866986dcbf76 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -373,6 +373,25 @@ struct LazySpecializationInfoLookupTable;
 
 } // namespace serialization
 
+struct VisibleLookupBlockOffsets {
+  uint64_t VisibleOffset = 0;
+  uint64_t ModuleLocalOffset = 0;
+  uint64_t TULocalOffset = 0;
+
+  operator bool() const {
+    return VisibleOffset || ModuleLocalOffset || // ModuleUnitLocalOffset ||
+           TULocalOffset;
+  }
+};
+
+struct LookupBlockOffsets : VisibleLookupBlockOffsets {
+  uint64_t LexicalOffset = 0;
+
+  operator bool() const {
+    return VisibleLookupBlockOffsets::operator bool() || LexicalOffset;
+  }
+};
+
 /// Reads an AST files chain containing the contents of a translation
 /// unit.
 ///
@@ -535,13 +554,6 @@ private:
   /// in the chain.
   DeclUpdateOffsetsMap DeclUpdateOffsets;
 
-  struct LookupBlockOffsets {
-    uint64_t LexicalOffset;
-    uint64_t VisibleOffset;
-    uint64_t ModuleLocalOffset;
-    uint64_t TULocalOffset;
-  };
-
   using DelayedNamespaceOffsetMapTy =
       llvm::DenseMap<GlobalDeclID, LookupBlockOffsets>;
 
diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index 07f7e8d919d8..ad1ec2673812 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -115,6 +115,13 @@ public:
     Record->push_back(BitOffset);
   }
 
+  void AddLookupOffsets(const LookupBlockOffsets &Offsets) {
+    AddOffset(Offsets.LexicalOffset);
+    AddOffset(Offsets.VisibleOffset);
+    AddOffset(Offsets.ModuleLocalOffset);
+    AddOffset(Offsets.TULocalOffset);
+  }
+
   /// Add the given statement or expression to the queue of
   /// statements to emit.
   ///
diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h
index 0f49646f3f02..97679ace8b61 100644
--- a/clang/include/clang/Serialization/ASTWriter.h
+++ b/clang/include/clang/Serialization/ASTWriter.h
@@ -75,6 +75,9 @@ class StoredDeclsList;
 class SwitchCase;
 class Token;
 
+struct VisibleLookupBlockOffsets;
+struct LookupBlockOffsets;
+
 namespace serialization {
 enum class DeclUpdateKind;
 } // namespace serialization
@@ -606,9 +609,7 @@ private:
   uint64_t WriteDeclContextLexicalBlock(ASTContext &Context,
                                         const DeclContext *DC);
   void WriteDeclContextVisibleBlock(ASTContext &Context, DeclContext *DC,
-                                    uint64_t &VisibleBlockOffset,
-                                    uint64_t &ModuleLocalBlockOffset,
-                                    uint64_t &TULocalBlockOffset);
+                                    VisibleLookupBlockOffsets &Offsets);
   void WriteTypeDeclOffsets();
   void WriteFileDeclIDsMap();
   void WriteComments(ASTContext &Context);
@@ -777,6 +778,9 @@ public:
     return (I == DeclIDs.end() || I->second >= clang::NUM_PREDEF_DECL_IDS);
   };
 
+  void AddLookupOffsets(const LookupBlockOffsets &Offsets,
+                        RecordDataImpl &Record);
+
   /// Emit a reference to a declaration.
   void AddDeclRef(const Decl *D, RecordDataImpl &Record);
   // Emit a reference to a declaration if the declaration was emitted.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index acda5a7c879d..a3fbc3d25aca 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -4097,8 +4097,8 @@ llvm::Error ASTReader::ReadASTBlock(ModuleFile &F,
         uint64_t TULocalOffset =
             TULocalLocalOffset ? BaseOffset + TULocalLocalOffset : 0;
 
-        DelayedNamespaceOffsetMap[ID] = {LexicalOffset, VisibleOffset,
-                                         ModuleLocalOffset, TULocalOffset};
+        DelayedNamespaceOffsetMap[ID] = {
+            {VisibleOffset, TULocalOffset, ModuleLocalOffset}, LexicalOffset};
 
         assert(!GetExistingDecl(ID) &&
                "We shouldn't load the namespace in the front of delayed "
@@ -8544,17 +8544,21 @@ bool ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
   SmallVector<NamedDecl *, 64> Decls;
   llvm::SmallPtrSet<NamedDecl *, 8> Found;
 
+  auto Find = [&, this](auto &&Table, auto &&Key) {
+    for (GlobalDeclID ID : Table.find(Key)) {
+      NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
+      if (ND->getDeclName() == Name && Found.insert(ND).second)
+        Decls.push_back(ND);
+    }
+  };
+
   Deserializing LookupResults(this);
 
   // FIXME: Clear the redundancy with templated lambda in C++20 when that's
   // available.
   if (auto It = Lookups.find(DC); It != Lookups.end()) {
     ++NumVisibleDeclContextsRead;
-    for (GlobalDeclID ID : It->second.Table.find(Name)) {
-      NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
-      if (ND->getDeclName() == Name && Found.insert(ND).second)
-        Decls.push_back(ND);
-    }
+    Find(It->second.Table, Name);
   }
 
   if (auto *NamedModule =
@@ -8562,21 +8566,13 @@ bool ASTReader::FindExternalVisibleDeclsByName(const DeclContext *DC,
                      : nullptr) {
     if (auto It = ModuleLocalLookups.find(DC); It != ModuleLocalLookups.end()) {
       ++NumModuleLocalVisibleDeclContexts;
-      for (GlobalDeclID ID : It->second.Table.find({Name, NamedModule})) {
-        NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
-        if (ND->getDeclName() == Name && Found.insert(ND).second)
-          Decls.push_back(ND);
-      }
+      Find(It->second.Table, std::make_pair(Name, NamedModule));
     }
   }
 
   if (auto It = TULocalLookups.find(DC); It != TULocalLookups.end()) {
     ++NumTULocalVisibleDeclContexts;
-    for (GlobalDeclID ID : It->second.Table.find(Name)) {
-      NamedDecl *ND = cast<NamedDecl>(GetDecl(ID));
-      if (ND->getDeclName() == Name && Found.insert(ND).second)
-        Decls.push_back(ND);
-    }
+    Find(It->second.Table, Name);
   }
 
   SetExternalVisibleDeclsForName(DC, Name, Decls);
diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index a1368a48351c..259c772e4222 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -414,9 +414,7 @@ public:
   void VisitOpenACCDeclareDecl(OpenACCDeclareDecl *D);
   void VisitOpenACCRoutineDecl(OpenACCRoutineDecl *D);
 
-  void VisitDeclContext(DeclContext *DC, uint64_t &LexicalOffset,
-                        uint64_t &VisibleOffset, uint64_t &ModuleLocalOffset,
-                        uint64_t &TULocalOffset);
+  void VisitDeclContext(DeclContext *DC, LookupBlockOffsets &Offsets);
 
   template <typename T>
   RedeclarableResult VisitRedeclarable(Redeclarable<T> *D);
@@ -1875,12 +1873,8 @@ void ASTDeclReader::VisitNamespaceDecl(NamespaceDecl *D) {
 
 void ASTDeclReader::VisitHLSLBufferDecl(HLSLBufferDecl *D) {
   VisitNamedDecl(D);
-  uint64_t LexicalOffset = 0;
-  uint64_t VisibleOffset = 0;
-  uint64_t ModuleLocalOffset = 0;
-  uint64_t TULocalOffset = 0;
-  VisitDeclContext(D, LexicalOffset, VisibleOffset, ModuleLocalOffset,
-                   TULocalOffset);
+  LookupBlockOffsets Offsets;
+  VisitDeclContext(D, Offsets);
   D->IsCBuffer = Record.readBool();
   D->KwLoc = readSourceLocation();
   D->LBraceLoc = readSourceLocation();
@@ -2794,14 +2788,12 @@ void ASTDeclReader::VisitLifetimeExtendedTemporaryDecl(
   mergeMergeable(D);
 }
 
-void ASTDeclReader::VisitDeclContext(DeclContext *DC, uint64_t &LexicalOffset,
-                                     uint64_t &VisibleOffset,
-                                     uint64_t &ModuleLocalOffset,
-                                     uint64_t &TULocalOffset) {
-  LexicalOffset = ReadLocalOffset();
-  VisibleOffset = ReadLocalOffset();
-  ModuleLocalOffset = ReadLocalOffset();
-  TULocalOffset = ReadLocalOffset();
+void ASTDeclReader::VisitDeclContext(DeclContext *DC,
+                                     LookupBlockOffsets &Offsets) {
+  Offsets.LexicalOffset = ReadLocalOffset();
+  Offsets.VisibleOffset = ReadLocalOffset();
+  Offsets.ModuleLocalOffset = ReadLocalOffset();
+  Offsets.TULocalOffset = ReadLocalOffset();
 }
 
 template <typename T>
@@ -4249,42 +4241,37 @@ Decl *ASTReader::ReadDeclRecord(GlobalDeclID ID) {
   // If this declaration is also a declaration context, get the
   // offsets for its tables of lexical and visible declarations.
   if (auto *DC = dyn_cast<DeclContext>(D)) {
-    uint64_t LexicalOffset = 0;
-    uint64_t VisibleOffset = 0;
-    uint64_t ModuleLocalOffset = 0;
-    uint64_t TULocalOffset = 0;
+    LookupBlockOffsets Offsets;
 
-    Reader.VisitDeclContext(DC, LexicalOffset, VisibleOffset, ModuleLocalOffset,
-                            TULocalOffset);
+    Reader.VisitDeclContext(DC, Offsets);
 
     // Get the lexical and visible block for the delayed namespace.
     // It is sufficient to judge if ID is in DelayedNamespaceOffsetMap.
     // But it may be more efficient to filter the other cases.
-    if (!LexicalOffset && !VisibleOffset && !ModuleLocalOffset &&
-        isa<NamespaceDecl>(D))
+    if (!Offsets && isa<NamespaceDecl>(D))
       if (auto Iter = DelayedNamespaceOffsetMap.find(ID);
-          Iter != DelayedNamespaceOffsetMap.end()) {
-        LexicalOffset = Iter->second.LexicalOffset;
-        VisibleOffset = Iter->second.VisibleOffset;
-        ModuleLocalOffset = Iter->second.ModuleLocalOffset;
-        TULocalOffset = Iter->second.TULocalOffset;
-      }
+          Iter != DelayedNamespaceOffsetMap.end())
+        Offsets = Iter->second;
 
-    if (LexicalOffset &&
-        ReadLexicalDeclContextStorage(*Loc.F, DeclsCursor, LexicalOffset, DC))
-      return nullptr;
-    if (VisibleOffset && ReadVisibleDeclContextStorage(
-                             *Loc.F, DeclsCursor, VisibleOffset, ID,
-                             VisibleDeclContextStorageKind::GenerallyVisible))
+    if (Offsets.VisibleOffset &&
+        ReadVisibleDeclContextStorage(
+            *Loc.F, DeclsCursor, Offsets.VisibleOffset, ID,
+            VisibleDeclContextStorageKind::GenerallyVisible))
       return nullptr;
-    if (ModuleLocalOffset &&
+    if (Offsets.ModuleLocalOffset &&
         ReadVisibleDeclContextStorage(
-            *Loc.F, DeclsCursor, ModuleLocalOffset, ID,
+            *Loc.F, DeclsCursor, Offsets.ModuleLocalOffset, ID,
             VisibleDeclContextStorageKind::ModuleLocalVisible))
       return nullptr;
-    if (TULocalOffset && ReadVisibleDeclContextStorage(
-                             *Loc.F, DeclsCursor, TULocalOffset, ID,
-                             VisibleDeclContextStorageKind::TULocalVisible))
+    if (Offsets.TULocalOffset &&
+        ReadVisibleDeclContextStorage(
+            *Loc.F, DeclsCursor, Offsets.TULocalOffset, ID,
+            VisibleDeclContextStorageKind::TULocalVisible))
+      return nullptr;
+
+    if (Offsets.LexicalOffset &&
+        ReadLexicalDeclContextStorage(*Loc.F, DeclsCursor,
+                                      Offsets.LexicalOffset, DC))
       return nullptr;
   }
   assert(Record.getIdx() == Record.size());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index be22ee522191..af7229d74887 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -4089,11 +4089,9 @@ public:
   using hash_value_type = unsigned;
   using offset_type = unsigned;
 
-protected:
   explicit ASTDeclContextNameLookupTraitBase(ASTWriter &Writer)
       : Writer(Writer) {}
 
-public:
   data_type getData(const DeclIDsTy &LocalIDs) {
     unsigned Start = DeclIDs.size();
     for (auto ID : LocalIDs)
@@ -4231,6 +4229,38 @@ public:
   }
 };
 
+class ASTDeclContextNameTrivialLookupTrait
+    : public ASTDeclContextNameLookupTraitBase {
+public:
+  using key_type = DeclarationNameKey;
+  using key_type_ref = key_type;
+
+public:
+  using ASTDeclContextNameLookupTraitBase::ASTDeclContextNameLookupTraitBase;
+
+  using ASTDeclContextNameLookupTraitBase::getData;
+
+  static bool EqualKey(key_type_ref a, key_type_ref b) { return a == b; }
+
+  hash_value_type ComputeHash(key_type Name) { return Name.getHash(); }
+
+  std::pair<unsigned, unsigned> EmitKeyDataLength(raw_ostream &Out,
+                                                  DeclarationNameKey Name,
+                                                  data_type_ref Lookup) {
+    auto [KeyLen, DataLen] = EmitKeyDataLengthBase(Out, Name, Lookup);
+    return emitULEBKeyDataLength(KeyLen, DataLen, Out);
+  }
+
+  void EmitKey(raw_ostream &Out, DeclarationNameKey Name, unsigned) {
+    return EmitKeyBase(Out, Name);
+  }
+
+  void EmitData(raw_ostream &Out, key_type_ref, data_type Lookup,
+                unsigned DataLen) {
+    EmitDataBase(Out, Lookup, DataLen);
+  }
+};
+
 static bool isModuleLocalDecl(NamedDecl *D) {
   // For decls not in a file context, they should have the same visibility
   // with their parent.
@@ -4273,25 +4303,43 @@ static bool isTULocalInNamedModules(NamedDecl *D) {
   return D->getLinkageInternal() == Linkage::Internal;
 }
 
-// Trait used for the on-disk hash table used in the method pool.
-template <bool CollectingTULocalDecls>
-class ASTDeclContextNameLookupTrait : public ASTDeclContextNameLookupTraitBase {
+class ASTDeclContextNameLookupTrait
+    : public ASTDeclContextNameTrivialLookupTrait {
 public:
+  using TULocalDeclsMapTy = llvm::DenseMap<key_type, DeclIDsTy>;
+
   using ModuleLevelDeclsMapTy =
       llvm::DenseMap<ModuleLevelNameLookupTrait::key_type, DeclIDsTy>;
 
-  using key_type = DeclarationNameKey;
-  using key_type_ref = key_type;
+private:
+  enum class LookupVisibility {
+    GenerallyVisibile,
+    // The decls can only be found by other TU in the same module.
+    // Note a clang::Module models a module unit instead of logical module
+    // in C++20.
+    ModuleLocalVisible,
+    // The decls can only be found by the TU itself that defines it.
+    TULocal,
+  };
 
-  using TULocalDeclsMapTy = llvm::DenseMap<key_type, DeclIDsTy>;
+  LookupVisibility getLookupVisibility(NamedDecl *D) const {
+    // Only named modules have other lookup visibility.
+    if (!Writer.isWritingStdCXXNamedModules())
+      return LookupVisibility::GenerallyVisibile;
+
+    if (isModuleLocalDecl(D))
+      return LookupVisibility::ModuleLocalVisible;
+    if (isTULocalInNamedModules(D))
+      return LookupVisibility::TULocal;
+    return LookupVisibility::GenerallyVisibile;
+  }
 
-private:
   ModuleLevelDeclsMapTy ModuleLocalDeclsMap;
   TULocalDeclsMapTy TULocalDeclsMap;
 
 public:
-  explicit ASTDeclContextNameLookupTrait(ASTWriter &Writer)
-      : ASTDeclContextNameLookupTraitBase(Writer) {}
+  using ASTDeclContextNameTrivialLookupTrait::
+      ASTDeclContextNameTrivialLookupTrait;
 
   template <typename Coll> data_type getData(const Coll &Decls) {
     unsigned Start = DeclIDs.size();
@@ -4312,7 +4360,8 @@ public:
 
       auto ID = Writer.GetDeclRef(DeclForLocalLookup);
 
-      if (isModuleLocalDecl(D)) {
+      switch (getLookupVisibility(DeclForLocalLookup)) {
+      case LookupVisibility::ModuleLocalVisible:
         if (UnsignedOrNone PrimaryModuleHash =
                 getPrimaryModuleHash(D->getOwningModule())) {
           auto Key = std::make_pair(D->getDeclName(), *PrimaryModuleHash);
@@ -4323,17 +4372,18 @@ public:
             Iter->second.push_back(ID);
           continue;
         }
+        break;
+      case LookupVisibility::TULocal: {
+        auto Iter = TULocalDeclsMap.find(D->getDeclName());
+        if (Iter == TULocalDeclsMap.end())
+          TULocalDeclsMap.insert({D->getDeclName(), DeclIDsTy{ID}});
+        else
+          Iter->second.push_back(ID);
+        continue;
       }
-
-      if constexpr (CollectingTULocalDecls) {
-        if (isTULocalInNamedModules(D)) {
-          auto Iter = TULocalDeclsMap.find(D->getDeclName());
-          if (Iter == TULocalDeclsMap.end())
-            TULocalDeclsMap.insert({D->getDeclName(), DeclIDsTy{ID}});
-          else
-            Iter->second.push_back(ID);
-          continue;
-        }
+      case LookupVisibility::GenerallyVisibile:
+        // Generally visible decls go into the general lookup table.
+        break;
       }
 
       DeclIDs.push_back(ID);
@@ -4341,33 +4391,11 @@ public:
     return std::make_pair(Start, DeclIDs.size());
   }
 
-  using ASTDeclContextNameLookupTraitBase::getData;
-
   const ModuleLevelDeclsMapTy &getModuleLocalDecls() {
     return ModuleLocalDeclsMap;
   }
 
   const TULocalDeclsMapTy &getTULocalDecls() { return TULocalDeclsMap; }
-
-  static bool EqualKey(key_type_ref a, key_type_ref b) { return a == b; }
-
-  hash_value_type ComputeHash(key_type Name) { return Name.getHash(); }
-
-  std::pair<unsigned, unsigned> EmitKeyDataLength(raw_ostream &Out,
-                                                  DeclarationNameKey Name,
-                                                  data_type_ref Lookup) {
-    auto [KeyLen, DataLen] = EmitKeyDataLengthBase(Out, Name, Lookup);
-    return emitULEBKeyDataLength(KeyLen, DataLen, Out);
-  }
-
-  void EmitKey(raw_ostream &Out, DeclarationNameKey Name, unsigned) {
-    return EmitKeyBase(Out, Name);
-  }
-
-  void EmitData(raw_ostream &Out, key_type_ref, data_type Lookup,
-                unsigned DataLen) {
-    EmitDataBase(Out, Lookup, DataLen);
-  }
 };
 
 } // namespace
@@ -4581,11 +4609,10 @@ void ASTWriter::GenerateNameLookupTable(
   assert(DC == DC->getPrimaryContext() && "only primary DC has lookup table");
 
   // Create the on-disk hash table representation.
-  MultiOnDiskHashTableGenerator<
-      reader::ASTDeclContextNameLookupTrait,
-      ASTDeclContextNameLookupTrait</*CollectingTULocal=*/true>>
+  MultiOnDiskHashTableGenerator<reader::ASTDeclContextNameLookupTrait,
+                                ASTDeclContextNameLookupTrait>
       Generator;
-  ASTDeclContextNameLookupTrait</*CollectingTULocal=*/true> Trait(*this);
+  ASTDeclContextNameLookupTrait Trait(*this);
 
   // The first step is to collect the declaration names which we need to
   // serialize into the name lookup table, and to collect them in a stable
@@ -4743,12 +4770,10 @@ void ASTWriter::GenerateNameLookupTable(
 
   const auto &TULocalDecls = Trait.getTULocalDecls();
   if (!TULocalDecls.empty() && !isGeneratingReducedBMI()) {
-    MultiOnDiskHashTableGenerator<
-        reader::ASTDeclContextNameLookupTrait,
-        ASTDeclContextNameLookupTrait</*CollectingTULocal=*/false>>
+    MultiOnDiskHashTableGenerator<reader::ASTDeclContextNameLookupTrait,
+                                  ASTDeclContextNameTrivialLookupTrait>
         TULookupGenerator;
-    ASTDeclContextNameLookupTrait</*CollectingTULocal=*/false> TULocalTrait(
-        *this);
+    ASTDeclContextNameTrivialLookupTrait TULocalTrait(*this);
 
     for (const auto &TULocalIter : TULocalDecls) {
       const auto &Key = TULocalIter.first;
@@ -4767,14 +4792,9 @@ void ASTWriter::GenerateNameLookupTable(
 ///
 /// \returns the offset of the DECL_CONTEXT_VISIBLE block within the
 /// bitstream, or 0 if no block was written.
-void ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
-                                             DeclContext *DC,
-                                             uint64_t &VisibleBlockOffset,
-                                             uint64_t &ModuleLocalBlockOffset,
-                                             uint64_t &TULocalBlockOffset) {
-  assert(VisibleBlockOffset == 0);
-  assert(ModuleLocalBlockOffset == 0);
-  assert(TULocalBlockOffset == 0);
+void ASTWriter::WriteDeclContextVisibleBlock(
+    ASTContext &Context, DeclContext *DC, VisibleLookupBlockOffsets &Offsets) {
+  assert(!Offsets);
 
   // If we imported a key declaration of this namespace, write the visible
   // lookup results as an update record for it rather than including them
@@ -4858,7 +4878,7 @@ void ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
   if (!Map || Map->empty())
     return;
 
-  VisibleBlockOffset = Stream.GetCurrentBitNo();
+  Offsets.VisibleOffset = Stream.GetCurrentBitNo();
   // Create the on-disk hash table in a buffer.
   SmallString<4096> LookupTable;
   SmallString<4096> ModuleLocalLookupTable;
@@ -4873,8 +4893,8 @@ void ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
   ++NumVisibleDeclContexts;
 
   if (!ModuleLocalLookupTable.empty()) {
-    ModuleLocalBlockOffset = Stream.GetCurrentBitNo();
-    assert(ModuleLocalBlockOffset > VisibleBlockOffset);
+    Offsets.ModuleLocalOffset = Stream.GetCurrentBitNo();
+    assert(Offsets.ModuleLocalOffset > Offsets.VisibleOffset);
     // Write the lookup table
     RecordData::value_type ModuleLocalRecord[] = {
         DECL_CONTEXT_MODULE_LOCAL_VISIBLE};
@@ -4884,7 +4904,7 @@ void ASTWriter::WriteDeclContextVisibleBlock(ASTContext &Context,
   }
 
   if (!TULookupTable.empty()) {
-    TULocalBlockOffset = Stream.GetCurrentBitNo();
+    Offsets.TULocalOffset = Stream.GetCurrentBitNo();
     // Write the lookup table
     RecordData::value_type TULocalDeclsRecord[] = {
         DECL_CONTEXT_TU_LOCAL_VISIBLE};
@@ -6203,31 +6223,26 @@ void ASTWriter::WriteDeclAndTypes(ASTContext &Context) {
   assert(DelayedNamespace.empty() || GeneratingReducedBMI);
   RecordData DelayedNamespaceRecord;
   for (NamespaceDecl *NS : DelayedNamespace) {
-    uint64_t LexicalOffset = WriteDeclContextLexicalBlock(Context, NS);
-    uint64_t VisibleOffset = 0;
-    uint64_t ModuleLocalOffset = 0;
-    uint64_t TULocalOffset = 0;
-    WriteDeclContextVisibleBlock(Context, NS, VisibleOffset, ModuleLocalOffset,
-                                 TULocalOffset);
+    LookupBlockOffsets Offsets;
 
-    // Write the offset relative to current block.
-    if (LexicalOffset)
-      LexicalOffset -= DeclTypesBlockStartOffset;
+    Offsets.LexicalOffset = WriteDeclContextLexicalBlock(Context, NS);
+    WriteDeclContextVisibleBlock(Context, NS, Offsets);
 
-    if (VisibleOffset)
-      VisibleOffset -= DeclTypesBlockStartOffset;
+    if (Offsets.LexicalOffset)
+      Offsets.LexicalOffset -= DeclTypesBlockStartOffset;
 
-    if (ModuleLocalOffset)
-      ModuleLocalOffset -= DeclTypesBlockStartOffset;
+    // Write the offset relative to current block.
+    if (Offsets.VisibleOffset)
+      Offsets.VisibleOffset -= DeclTypesBlockStartOffset;
 
-    if (TULocalOffset)
-      TULocalOffset -= DeclTypesBlockStartOffset;
+    if (Offsets.ModuleLocalOffset)
+      Offsets.ModuleLocalOffset -= DeclTypesBlockStartOffset;
+
+    if (Offsets.TULocalOffset)
+      Offsets.TULocalOffset -= DeclTypesBlockStartOffset;
 
     AddDeclRef(NS, DelayedNamespaceRecord);
-    DelayedNamespaceRecord.push_back(LexicalOffset);
-    DelayedNamespaceRecord.push_back(VisibleOffset);
-    DelayedNamespaceRecord.push_back(ModuleLocalOffset);
-    DelayedNamespaceRecord.push_back(TULocalOffset);
+    AddLookupOffsets(Offsets, DelayedNamespaceRecord);
   }
 
   // The process of writing lexical and visible block for delayed namespace
@@ -6818,6 +6833,14 @@ TypeID ASTWriter::GetOrCreateTypeID(ASTContext &Context, QualType T) {
   });
 }
 
+void ASTWriter::AddLookupOffsets(const LookupBlockOffsets &Offsets,
+                                 RecordDataImpl &Record) {
+  Record.push_back(Offsets.LexicalOffset);
+  Record.push_back(Offsets.VisibleOffset);
+  Record.push_back(Offsets.ModuleLocalOffset);
+  Record.push_back(Offsets.TULocalOffset);
+}
+
 void ASTWriter::AddEmittedDeclRef(const Decl *D, RecordDataImpl &Record) {
   if (!wasDeclEmitted(D))
     return;
diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp
index 052cb5a253bf..2d93832a9ac3 100644
--- a/clang/lib/Serialization/ASTWriterDecl.cpp
+++ b/clang/lib/Serialization/ASTWriterDecl.cpp
@@ -2189,11 +2189,7 @@ void ASTDeclWriter::VisitDeclContext(DeclContext *DC) {
   static_assert(DeclContext::NumDeclContextBits == 13,
                 "You need to update the serializer after you change the "
                 "DeclContextBits");
-
-  uint64_t LexicalOffset = 0;
-  uint64_t VisibleOffset = 0;
-  uint64_t ModuleLocalOffset = 0;
-  uint64_t TULocalOffset = 0;
+  LookupBlockOffsets Offsets;
 
   if (Writer.isGeneratingReducedBMI() && isa<NamespaceDecl>(DC) &&
       cast<NamespaceDecl>(DC)->isFromExplicitGlobalModule()) {
@@ -2202,17 +2198,12 @@ void ASTDeclWriter::VisitDeclContext(DeclContext *DC) {
     // details.
     Writer.DelayedNamespace.push_back(cast<NamespaceDecl>(DC));
   } else {
-    LexicalOffset =
+    Offsets.LexicalOffset =
         Writer.WriteDeclContextLexicalBlock(Record.getASTContext(), DC);
-    Writer.WriteDeclContextVisibleBlock(Record.getASTContext(), DC,
-                                        VisibleOffset, ModuleLocalOffset,
-                                        TULocalOffset);
+    Writer.WriteDeclContextVisibleBlock(Record.getASTContext(), DC, Offsets);
   }
 
-  Record.AddOffset(LexicalOffset);
-  Record.AddOffset(VisibleOffset);
-  Record.AddOffset(ModuleLocalOffset);
-  Record.AddOffset(TULocalOffset);
+  Record.AddLookupOffsets(Offsets);
 }
 
 const Decl *ASTWriter::getFirstLocalDecl(const Decl *D) {
-- 
cgit v1.2.3


From 5bee2c34bde1aa8b0fb5aed7d5ce330f094f6436 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 19 Jun 2025 19:02:42 +0900
Subject: RuntimeLibcalls: Pass in FloatABI and EABI type (#144691)

We need the full set of ABI options to accurately compute
the full set of libcalls. This partially resolves missing
information required to compute the set of ARM calls.
---
 llvm/include/llvm/IR/RuntimeLibcalls.h  | 10 +++++++---
 llvm/lib/CodeGen/TargetLoweringBase.cpp |  3 ++-
 llvm/lib/IR/RuntimeLibcalls.cpp         | 18 +++++++++++++++---
 llvm/lib/Object/IRSymtab.cpp            |  2 +-
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 10 ----------
 5 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index 3e1531ebfd9d..a6a180f5ed8d 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -19,6 +19,7 @@
 #include "llvm/IR/CallingConv.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/Support/AtomicOrdering.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
 
@@ -53,8 +54,10 @@ static inline auto libcalls() {
 
 /// A simple container for information about the supported runtime calls.
 struct RuntimeLibcallsInfo {
-  explicit RuntimeLibcallsInfo(const Triple &TT) {
-    initLibcalls(TT);
+  explicit RuntimeLibcallsInfo(const Triple &TT,
+                               FloatABI::ABIType FloatABI = FloatABI::Default,
+                               EABI EABIVersion = EABI::Default) {
+    initLibcalls(TT, FloatABI, EABIVersion);
   }
 
   /// Rename the default libcall routine name for the specified libcall.
@@ -144,7 +147,8 @@ private:
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
   /// should be placed here.
-  LLVM_ABI void initLibcalls(const Triple &TT);
+  LLVM_ABI void initLibcalls(const Triple &TT, FloatABI::ABIType FloatABI,
+                             EABI ABIType);
 };
 
 } // namespace RTLIB
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index b1afdc2a3ac3..2b5087cd38f5 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -632,7 +632,8 @@ void RTLIB::initCmpLibcallCCs(ISD::CondCode *CmpLibcallCCs) {
 
 /// NOTE: The TargetMachine owns TLOF.
 TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm)
-    : TM(tm), Libcalls(TM.getTargetTriple()) {
+    : TM(tm), Libcalls(TM.getTargetTriple(), TM.Options.FloatABIType,
+                       TM.Options.EABIVersion) {
   initActions();
 
   // Perform these initializations only once.
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index a57b08919346..74dccdf172d4 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -65,7 +65,17 @@ static void setAArch64LibcallNames(RuntimeLibcallsInfo &Info,
 #undef LCALLNAME5
 }
 
-static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT) {
+static void setARMLibcallNames(RuntimeLibcallsInfo &Info, const Triple &TT,
+                               FloatABI::ABIType FloatABIType,
+                               EABI EABIVersion) {
+  if (!TT.isOSDarwin() && !TT.isiOS() && !TT.isWatchOS() && !TT.isDriverKit()) {
+    CallingConv::ID DefaultCC = FloatABIType == FloatABI::Hard
+                                    ? CallingConv::ARM_AAPCS_VFP
+                                    : CallingConv::ARM_AAPCS;
+    for (RTLIB::Libcall LC : RTLIB::libcalls())
+      Info.setLibcallCallingConv(LC, DefaultCC);
+  }
+
   // Register based DivRem for AEABI (RTABI 4.2)
   if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
       TT.isTargetMuslAEABI() || TT.isOSWindows()) {
@@ -346,7 +356,9 @@ static void setLongDoubleIsF128Libm(RuntimeLibcallsInfo &Info,
 
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
-void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
+void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
+                                       FloatABI::ABIType FloatABI,
+                                       EABI EABIVersion) {
   initSoftFloatCmpLibcallPredicates();
 
   initSoftFloatCmpLibcallPredicates();
@@ -539,7 +551,7 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT) {
   if (TT.isAArch64())
     setAArch64LibcallNames(*this, TT);
   else if (TT.isARM() || TT.isThumb())
-    setARMLibcallNames(*this, TT);
+    setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
   else if (TT.getArch() == Triple::ArchType::avr) {
     // Division rtlib functions (not supported), use divmod functions instead
     setLibcallName(RTLIB::SDIV_I8, nullptr);
diff --git a/llvm/lib/Object/IRSymtab.cpp b/llvm/lib/Object/IRSymtab.cpp
index 806477ae3de0..494ec089d7bd 100644
--- a/llvm/lib/Object/IRSymtab.cpp
+++ b/llvm/lib/Object/IRSymtab.cpp
@@ -216,7 +216,7 @@ Expected<int> Builder::getComdatIndex(const Comdat *C, const Module *M) {
 static DenseSet<StringRef> buildPreservedSymbolsSet(const Triple &TT) {
   DenseSet<StringRef> PreservedSymbolSet(std::begin(PreservedSymbols),
                                          std::end(PreservedSymbols));
-
+  // FIXME: Do we need to pass in ABI fields from TargetOptions?
   RTLIB::RuntimeLibcallsInfo Libcalls(TT);
   for (const char *Name : Libcalls.getLibcallNames()) {
     if (Name)
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 6e653687dbcb..91fb7bc4578b 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -515,16 +515,6 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
-  if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() &&
-      !Subtarget->isTargetWatchOS() && !Subtarget->isTargetDriverKit()) {
-    bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard;
-
-    for (RTLIB::Libcall LC : RTLIB::libcalls()) {
-      setLibcallCallingConv(LC, IsHFTarget ? CallingConv::ARM_AAPCS_VFP
-                                           : CallingConv::ARM_AAPCS);
-    }
-  }
-
   if (Subtarget->isTargetMachO()) {
     // Uses VFP for Thumb libfuncs if available.
     if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
-- 
cgit v1.2.3


From 305953a32ded8a43b22f65cf73d9214729feb1fc Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 19 Jun 2025 19:05:56 +0900
Subject: MC: Move ExceptionHandling enum to Support (#144692)

Similar to b5967264b0fbfd502b3a7edec27409e966fb68be, we need
to use this in RuntimeLibcalls to compute the set of library
calls.
---
 llvm/include/llvm/MC/MCTargetOptions.h | 13 +------------
 llvm/include/llvm/Support/CodeGen.h    | 12 ++++++++++++
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/MC/MCTargetOptions.h b/llvm/include/llvm/MC/MCTargetOptions.h
index 3ee21d9cda4b..d95adf92b9a8 100644
--- a/llvm/include/llvm/MC/MCTargetOptions.h
+++ b/llvm/include/llvm/MC/MCTargetOptions.h
@@ -10,6 +10,7 @@
 #define LLVM_MC_MCTARGETOPTIONS_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/Support/CodeGen.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Compression.h"
 #include <string>
@@ -17,18 +18,6 @@
 
 namespace llvm {
 
-enum class ExceptionHandling {
-  None,     ///< No exception support
-  DwarfCFI, ///< DWARF-like instruction based exceptions
-  SjLj,     ///< setjmp/longjmp based exceptions
-  ARM,      ///< ARM EHABI
-  WinEH,    ///< Windows Exception Handling
-  Wasm,     ///< WebAssembly Exception Handling
-  AIX,      ///< AIX Exception Handling
-  ZOS,      ///< z/OS MVS Exception Handling. Very similar to DwarfCFI, but the PPA1
-            ///< is used instead of an .eh_frame section.
-};
-
 enum class EmitDwarfUnwindType {
   Always,          // Always emit dwarf unwind
   NoCompactUnwind, // Only emit if compact unwind isn't available
diff --git a/llvm/include/llvm/Support/CodeGen.h b/llvm/include/llvm/Support/CodeGen.h
index 90733b50385a..cd1f9167b996 100644
--- a/llvm/include/llvm/Support/CodeGen.h
+++ b/llvm/include/llvm/Support/CodeGen.h
@@ -50,6 +50,18 @@ namespace llvm {
     };
   }
 
+  enum class ExceptionHandling {
+    None,     ///< No exception support
+    DwarfCFI, ///< DWARF-like instruction based exceptions
+    SjLj,     ///< setjmp/longjmp based exceptions
+    ARM,      ///< ARM EHABI
+    WinEH,    ///< Windows Exception Handling
+    Wasm,     ///< WebAssembly Exception Handling
+    AIX,      ///< AIX Exception Handling
+    ZOS, ///< z/OS MVS Exception Handling. Very similar to DwarfCFI, but the
+         ///< PPA1 is used instead of an .eh_frame section.
+  };
+
   namespace FloatABI {
   enum ABIType {
     Default, // Target-specific (either soft or hard depending on triple, etc).
-- 
cgit v1.2.3


From 1c35fe4e6b2596d153da82b23d04a3779fb12730 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 19 Jun 2025 19:08:52 +0900
Subject: RuntimeLibcalls: Pass in exception handling type (#144696)

All of the ABI options that influence libcall decisions need
to be passed in.
---
 llvm/include/llvm/IR/RuntimeLibcalls.h  | 14 ++++++++------
 llvm/lib/CodeGen/TargetLoweringBase.cpp |  4 ++--
 llvm/lib/IR/RuntimeLibcalls.cpp         |  6 ++++++
 llvm/lib/Target/VE/VEISelLowering.cpp   |  2 --
 llvm/lib/Target/X86/X86ISelLowering.cpp |  4 ----
 5 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index a6a180f5ed8d..71f38bedf17e 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -54,10 +54,12 @@ static inline auto libcalls() {
 
 /// A simple container for information about the supported runtime calls.
 struct RuntimeLibcallsInfo {
-  explicit RuntimeLibcallsInfo(const Triple &TT,
-                               FloatABI::ABIType FloatABI = FloatABI::Default,
-                               EABI EABIVersion = EABI::Default) {
-    initLibcalls(TT, FloatABI, EABIVersion);
+  explicit RuntimeLibcallsInfo(
+      const Triple &TT,
+      ExceptionHandling ExceptionModel = ExceptionHandling::None,
+      FloatABI::ABIType FloatABI = FloatABI::Default,
+      EABI EABIVersion = EABI::Default) {
+    initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion);
   }
 
   /// Rename the default libcall routine name for the specified libcall.
@@ -147,8 +149,8 @@ private:
 
   /// Set default libcall names. If a target wants to opt-out of a libcall it
   /// should be placed here.
-  LLVM_ABI void initLibcalls(const Triple &TT, FloatABI::ABIType FloatABI,
-                             EABI ABIType);
+  LLVM_ABI void initLibcalls(const Triple &TT, ExceptionHandling ExceptionModel,
+                             FloatABI::ABIType FloatABI, EABI ABIType);
 };
 
 } // namespace RTLIB
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 2b5087cd38f5..41e73b853093 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -632,8 +632,8 @@ void RTLIB::initCmpLibcallCCs(ISD::CondCode *CmpLibcallCCs) {
 
 /// NOTE: The TargetMachine owns TLOF.
 TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm)
-    : TM(tm), Libcalls(TM.getTargetTriple(), TM.Options.FloatABIType,
-                       TM.Options.EABIVersion) {
+    : TM(tm), Libcalls(TM.getTargetTriple(), TM.Options.ExceptionModel,
+                       TM.Options.FloatABIType, TM.Options.EABIVersion) {
   initActions();
 
   // Perform these initializations only once.
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 74dccdf172d4..ad2904d6d2ea 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -357,6 +357,7 @@ static void setLongDoubleIsF128Libm(RuntimeLibcallsInfo &Info,
 /// Set default libcall names. If a target wants to opt-out of a libcall it
 /// should be placed here.
 void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
+                                       ExceptionHandling ExceptionModel,
                                        FloatABI::ABIType FloatABI,
                                        EABI EABIVersion) {
   initSoftFloatCmpLibcallPredicates();
@@ -373,6 +374,11 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
   if (TT.isX86() && TT.isGNUEnvironment())
     setLongDoubleIsF128Libm(*this, /*FiniteOnlyFuncs=*/true);
 
+  if (TT.isX86() || TT.isVE()) {
+    if (ExceptionModel == ExceptionHandling::SjLj)
+      setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
+  }
+
   // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf".
   if (TT.isPPC()) {
     setLibcallName(RTLIB::ADD_F128, "__addkf3");
diff --git a/llvm/lib/Target/VE/VEISelLowering.cpp b/llvm/lib/Target/VE/VEISelLowering.cpp
index b5a0d26abbf8..98c5fdd13898 100644
--- a/llvm/lib/Target/VE/VEISelLowering.cpp
+++ b/llvm/lib/Target/VE/VEISelLowering.cpp
@@ -298,8 +298,6 @@ void VETargetLowering::initSPUActions() {
   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
   setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
-  if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
-    setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
   /// } SJLJ instructions
 
   // Intrinsic instructions
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 4751361c71f2..defb7730b4c7 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -513,10 +513,6 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
   setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
   setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
 
-  // FIXME: This should be set in RuntimeLibcallsInfo
-  if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
-    setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
-
   // Darwin ABI issue.
   for (auto VT : { MVT::i32, MVT::i64 }) {
     if (VT == MVT::i64 && !Subtarget.is64Bit())
-- 
cgit v1.2.3


From 74054cab7a3e04b323828850409343932e975737 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Thu, 19 Jun 2025 12:29:58 +0100
Subject: [HashRecognize] Make it a non-PM analysis (#144742)

Make HashRecognize a non-PassManager analysis that can be called to get
the result on-demand, creating a new getResult() entry-point. The issue
was discovered when attempting to use the analysis to perform a
transform in LoopIdiomRecognize.
---
 llvm/include/llvm/Analysis/HashRecognize.h | 14 +++-----------
 llvm/lib/Analysis/HashRecognize.cpp        | 18 +++++++++---------
 llvm/lib/Passes/PassRegistry.def           |  1 -
 3 files changed, 12 insertions(+), 21 deletions(-)

diff --git a/llvm/include/llvm/Analysis/HashRecognize.h b/llvm/include/llvm/Analysis/HashRecognize.h
index c169383bf7b0..0361dfcd2352 100644
--- a/llvm/include/llvm/Analysis/HashRecognize.h
+++ b/llvm/include/llvm/Analysis/HashRecognize.h
@@ -84,12 +84,13 @@ class HashRecognize {
 public:
   HashRecognize(const Loop &L, ScalarEvolution &SE);
 
-  // The main analysis entry point.
+  // The main analysis entry points.
   std::variant<PolynomialInfo, ErrBits, StringRef> recognizeCRC() const;
+  std::optional<PolynomialInfo> getResult() const;
 
   // Auxilary entry point after analysis to interleave the generating polynomial
   // and return a 256-entry CRC table.
-  CRCTable genSarwateTable(const APInt &GenPoly, bool ByteOrderSwapped) const;
+  static CRCTable genSarwateTable(const APInt &GenPoly, bool ByteOrderSwapped);
 
   void print(raw_ostream &OS) const;
 
@@ -107,15 +108,6 @@ public:
   PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &);
 };
-
-class HashRecognizeAnalysis : public AnalysisInfoMixin<HashRecognizeAnalysis> {
-  friend AnalysisInfoMixin<HashRecognizeAnalysis>;
-  static AnalysisKey Key;
-
-public:
-  using Result = HashRecognize;
-  Result run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR);
-};
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Analysis/HashRecognize.cpp b/llvm/lib/Analysis/HashRecognize.cpp
index d11602f92187..1a5dfbe4bed6 100644
--- a/llvm/lib/Analysis/HashRecognize.cpp
+++ b/llvm/lib/Analysis/HashRecognize.cpp
@@ -472,7 +472,7 @@ static bool checkExtractBits(const KnownBits &Known, unsigned N,
 /// polynomial. The optimization technique of table-lookup for CRC is also
 /// called the Sarwate algorithm.
 CRCTable HashRecognize::genSarwateTable(const APInt &GenPoly,
-                                        bool ByteOrderSwapped) const {
+                                        bool ByteOrderSwapped) {
   unsigned BW = GenPoly.getBitWidth();
   CRCTable Table;
   Table[0] = APInt::getZero(BW);
@@ -686,6 +686,13 @@ void HashRecognize::print(raw_ostream &OS) const {
 void HashRecognize::dump() const { print(dbgs()); }
 #endif
 
+std::optional<PolynomialInfo> HashRecognize::getResult() const {
+  auto Res = HashRecognize(L, SE).recognizeCRC();
+  if (std::holds_alternative<PolynomialInfo>(Res))
+    return std::get<PolynomialInfo>(Res);
+  return std::nullopt;
+}
+
 HashRecognize::HashRecognize(const Loop &L, ScalarEvolution &SE)
     : L(L), SE(SE) {}
 
@@ -693,13 +700,6 @@ PreservedAnalyses HashRecognizePrinterPass::run(Loop &L,
                                                 LoopAnalysisManager &AM,
                                                 LoopStandardAnalysisResults &AR,
                                                 LPMUpdater &) {
-  AM.getResult<HashRecognizeAnalysis>(L, AR).print(OS);
+  HashRecognize(L, AR.SE).print(OS);
   return PreservedAnalyses::all();
 }
-
-HashRecognize HashRecognizeAnalysis::run(Loop &L, LoopAnalysisManager &AM,
-                                         LoopStandardAnalysisResults &AR) {
-  return {L, AR.SE};
-}
-
-AnalysisKey HashRecognizeAnalysis::Key;
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index f761d0dab09a..ec14c6a9211d 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -672,7 +672,6 @@ LOOPNEST_PASS("no-op-loopnest", NoOpLoopNestPass())
 #define LOOP_ANALYSIS(NAME, CREATE_PASS)
 #endif
 LOOP_ANALYSIS("ddg", DDGAnalysis())
-LOOP_ANALYSIS("hash-recognize", HashRecognizeAnalysis())
 LOOP_ANALYSIS("iv-users", IVUsersAnalysis())
 LOOP_ANALYSIS("no-op-loop", NoOpLoopAnalysis())
 LOOP_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC))
-- 
cgit v1.2.3


From 30824c449a893771c3f25f0eb29cfa9d2cfd4f15 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Thu, 19 Jun 2025 12:48:39 +0100
Subject: [lldb][DWARFASTParserClang] GetCXXObjectParameter to take DeclContext
 DIE parameter (#144876)

I'm trying to call `GetCXXObjectParameter` from unit-tests in a
follow-up patch and taking a `DWARFDIE` instead of `clang::DeclContext`
makes that much simpler. These should be equivalent, since all we're
trying to check is that the parent context is a record type.
---
 .../Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp   | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 620501b304e6..7fc1d70898d1 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -163,14 +163,14 @@ static bool TagIsRecordType(dw_tag_t tag) {
 /// a default DWARFDIE. If \c containing_decl_ctx is not a valid
 /// C++ declaration context for class methods, assume no object
 /// parameter exists for the given \c subprogram.
-static DWARFDIE
-GetCXXObjectParameter(const DWARFDIE &subprogram,
-                      const clang::DeclContext &containing_decl_ctx) {
+static DWARFDIE GetCXXObjectParameter(const DWARFDIE &subprogram,
+                                      const DWARFDIE &decl_ctx_die) {
+  assert(subprogram);
   assert(subprogram.Tag() == DW_TAG_subprogram ||
          subprogram.Tag() == DW_TAG_inlined_subroutine ||
          subprogram.Tag() == DW_TAG_subroutine_type);
 
-  if (!DeclKindIsCXXClass(containing_decl_ctx.getDeclKind()))
+  if (!decl_ctx_die.IsStructUnionOrClass())
     return {};
 
   if (DWARFDIE object_parameter =
@@ -1304,8 +1304,7 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
   clang::CallingConv calling_convention =
       ConvertDWARFCallingConventionToClang(attrs);
 
-  const DWARFDIE object_parameter =
-      GetCXXObjectParameter(die, *containing_decl_ctx);
+  const DWARFDIE object_parameter = GetCXXObjectParameter(die, decl_ctx_die);
 
   // clang_type will get the function prototype clang type after this
   // call
@@ -2411,12 +2410,13 @@ DWARFASTParserClang::ConstructDemangledNameFromDWARF(const DWARFDIE &die) {
   DWARFDeclContext decl_ctx = die.GetDWARFDeclContext();
   sstr << decl_ctx.GetQualifiedName();
 
+  DWARFDIE decl_ctx_die;
   clang::DeclContext *containing_decl_ctx =
-      GetClangDeclContextContainingDIE(die, nullptr);
+      GetClangDeclContextContainingDIE(die, &decl_ctx_die);
   assert(containing_decl_ctx);
 
-  const unsigned cv_quals = GetCXXMethodCVQuals(
-      die, GetCXXObjectParameter(die, *containing_decl_ctx));
+  const unsigned cv_quals =
+      GetCXXMethodCVQuals(die, GetCXXObjectParameter(die, decl_ctx_die));
 
   ParseChildParameters(containing_decl_ctx, die, is_variadic,
                        has_template_params, param_types, param_names);
-- 
cgit v1.2.3


From 046e2f545ef568b2ce577c9172a0f147dc376071 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 19 Jun 2025 12:49:45 +0100
Subject: [LV] Add interleaving test with partial reductions and non-const
 start.

Add test coverage for mis-compile after
https://github.com/llvm/llvm-project/pull/142290.
---
 .../AArch64/partial-reduce-interleave.ll           | 154 +++++++++++++++++++++
 1 file changed, 154 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll

diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll
new file mode 100644
index 000000000000..b5bc7bf80372
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll
@@ -0,0 +1,154 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -p loop-vectorize -force-vector-width=16 -force-vector-interleave=2 -mattr=+dotprod -S %s | FileCheck --check-prefix=IC2 %s
+; RUN: opt -p loop-vectorize -force-vector-width=16 -force-vector-interleave=4 -mattr=+dotprod -S %s | FileCheck --check-prefix=IC4 %s
+
+target triple = "arm64-apple-macosx"
+
+define i32 @partial_reduce_with_non_constant_start_value(ptr %src, i32 %rdx.start, i64 %n) {
+; IC2-LABEL: define i32 @partial_reduce_with_non_constant_start_value(
+; IC2-SAME: ptr [[SRC:%.*]], i32 [[RDX_START:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; IC2-NEXT:  [[ENTRY:.*]]:
+; IC2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 32
+; IC2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC2:       [[VECTOR_PH]]:
+; IC2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 32
+; IC2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; IC2-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[RDX_START]], i32 0
+; IC2-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC2:       [[VECTOR_BODY]]:
+; IC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; IC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; IC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
+; IC2-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; IC2-NEXT:    [[WIDE_LOAD2:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; IC2-NEXT:    [[TMP4:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; IC2-NEXT:    [[TMP5:%.*]] = zext <16 x i8> [[WIDE_LOAD2]] to <16 x i32>
+; IC2-NEXT:    [[TMP6:%.*]] = mul nuw nsw <16 x i32> [[TMP4]], [[TMP4]]
+; IC2-NEXT:    [[TMP7:%.*]] = mul nuw nsw <16 x i32> [[TMP5]], [[TMP5]]
+; IC2-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP6]])
+; IC2-NEXT:    [[PARTIAL_REDUCE3]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP7]])
+; IC2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
+; IC2-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC2-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC2:       [[MIDDLE_BLOCK]]:
+; IC2-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE3]], [[PARTIAL_REDUCE]]
+; IC2-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX]])
+; IC2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; IC2-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC2:       [[SCALAR_PH]]:
+; IC2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC2-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP9]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[ENTRY]] ]
+; IC2-NEXT:    br label %[[LOOP:.*]]
+; IC2:       [[LOOP]]:
+; IC2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC2-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; IC2-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; IC2-NEXT:    [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; IC2-NEXT:    [[CONV:%.*]] = zext i8 [[L]] to i32
+; IC2-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
+; IC2-NEXT:    [[RDX_NEXT]] = add nsw i32 [[MUL]], [[RDX]]
+; IC2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; IC2-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IC2-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC2:       [[EXIT]]:
+; IC2-NEXT:    [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP9]], %[[MIDDLE_BLOCK]] ]
+; IC2-NEXT:    ret i32 [[RDX_NEXT_LCSSA]]
+;
+; IC4-LABEL: define i32 @partial_reduce_with_non_constant_start_value(
+; IC4-SAME: ptr [[SRC:%.*]], i32 [[RDX_START:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; IC4-NEXT:  [[ENTRY:.*]]:
+; IC4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 64
+; IC4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IC4:       [[VECTOR_PH]]:
+; IC4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], 64
+; IC4-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
+; IC4-NEXT:    [[TMP0:%.*]] = insertelement <4 x i32> zeroinitializer, i32 [[RDX_START]], i32 0
+; IC4-NEXT:    br label %[[VECTOR_BODY:.*]]
+; IC4:       [[VECTOR_BODY]]:
+; IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
+; IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
+; IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
+; IC4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 32
+; IC4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 48
+; IC4-NEXT:    [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1
+; IC4-NEXT:    [[WIDE_LOAD4:%.*]] = load <16 x i8>, ptr [[TMP3]], align 1
+; IC4-NEXT:    [[WIDE_LOAD5:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1
+; IC4-NEXT:    [[WIDE_LOAD6:%.*]] = load <16 x i8>, ptr [[TMP5]], align 1
+; IC4-NEXT:    [[TMP6:%.*]] = zext <16 x i8> [[WIDE_LOAD]] to <16 x i32>
+; IC4-NEXT:    [[TMP7:%.*]] = zext <16 x i8> [[WIDE_LOAD4]] to <16 x i32>
+; IC4-NEXT:    [[TMP8:%.*]] = zext <16 x i8> [[WIDE_LOAD5]] to <16 x i32>
+; IC4-NEXT:    [[TMP9:%.*]] = zext <16 x i8> [[WIDE_LOAD6]] to <16 x i32>
+; IC4-NEXT:    [[TMP10:%.*]] = mul nuw nsw <16 x i32> [[TMP6]], [[TMP6]]
+; IC4-NEXT:    [[TMP11:%.*]] = mul nuw nsw <16 x i32> [[TMP7]], [[TMP7]]
+; IC4-NEXT:    [[TMP12:%.*]] = mul nuw nsw <16 x i32> [[TMP8]], [[TMP8]]
+; IC4-NEXT:    [[TMP13:%.*]] = mul nuw nsw <16 x i32> [[TMP9]], [[TMP9]]
+; IC4-NEXT:    [[PARTIAL_REDUCE]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI]], <16 x i32> [[TMP10]])
+; IC4-NEXT:    [[PARTIAL_REDUCE7]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI1]], <16 x i32> [[TMP11]])
+; IC4-NEXT:    [[PARTIAL_REDUCE8]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI2]], <16 x i32> [[TMP12]])
+; IC4-NEXT:    [[PARTIAL_REDUCE9]] = call <4 x i32> @llvm.experimental.vector.partial.reduce.add.v4i32.v16i32(<4 x i32> [[VEC_PHI3]], <16 x i32> [[TMP13]])
+; IC4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 64
+; IC4-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; IC4-NEXT:    br i1 [[TMP18]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; IC4:       [[MIDDLE_BLOCK]]:
+; IC4-NEXT:    [[BIN_RDX:%.*]] = add <4 x i32> [[PARTIAL_REDUCE7]], [[PARTIAL_REDUCE]]
+; IC4-NEXT:    [[BIN_RDX10:%.*]] = add <4 x i32> [[PARTIAL_REDUCE8]], [[BIN_RDX]]
+; IC4-NEXT:    [[BIN_RDX11:%.*]] = add <4 x i32> [[PARTIAL_REDUCE9]], [[BIN_RDX10]]
+; IC4-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[BIN_RDX11]])
+; IC4-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
+; IC4-NEXT:    br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]]
+; IC4:       [[SCALAR_PH]]:
+; IC4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ]
+; IC4-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ [[RDX_START]], %[[ENTRY]] ]
+; IC4-NEXT:    br label %[[LOOP:.*]]
+; IC4:       [[LOOP]]:
+; IC4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
+; IC4-NEXT:    [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], %[[SCALAR_PH]] ], [ [[RDX_NEXT:%.*]], %[[LOOP]] ]
+; IC4-NEXT:    [[GEP_SRC:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[IV]]
+; IC4-NEXT:    [[L:%.*]] = load i8, ptr [[GEP_SRC]], align 1
+; IC4-NEXT:    [[CONV:%.*]] = zext i8 [[L]] to i32
+; IC4-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[CONV]], [[CONV]]
+; IC4-NEXT:    [[RDX_NEXT]] = add nsw i32 [[MUL]], [[RDX]]
+; IC4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; IC4-NEXT:    [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]]
+; IC4-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; IC4:       [[EXIT]]:
+; IC4-NEXT:    [[RDX_NEXT_LCSSA:%.*]] = phi i32 [ [[RDX_NEXT]], %[[LOOP]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ]
+; IC4-NEXT:    ret i32 [[RDX_NEXT_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ]
+  %rdx = phi i32 [ %rdx.start, %entry ], [ %rdx.next, %loop ]
+  %gep.src = getelementptr inbounds i8, ptr %src, i64 %iv
+  %l = load i8, ptr %gep.src, align 1
+  %conv = zext i8 %l to i32
+  %mul = mul nuw nsw i32 %conv, %conv
+  %rdx.next = add nsw i32 %mul, %rdx
+  %iv.next = add nsw i64 %iv, 1
+  %ec = icmp eq i64 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  ret i32 %rdx.next
+}
+;.
+; IC2: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; IC2: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; IC2: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; IC2: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
+; IC4: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; IC4: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; IC4: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; IC4: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
-- 
cgit v1.2.3


From e33f13ba4824d807e846e7783a48efd6c0bf58ee Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 19 Jun 2025 13:59:22 +0200
Subject: [mlir][arith] Add overflow flags to `arith.trunci` (#144863)

LLVM already supports overflow flags on `llvm.trunc` for a while. This
commit adds support for these flags to `arith.trunci`.
---
 mlir/include/mlir/Dialect/Arith/IR/ArithOps.td     | 36 ++++++++++++++++++----
 mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp    |  3 +-
 mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td | 12 ++++----
 .../test/Conversion/ArithToLLVM/arith-to-llvm.mlir |  2 ++
 mlir/test/Dialect/Arith/ops.mlir                   |  2 ++
 5 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index adc27ae6bdaf..993f36f556e8 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -226,7 +226,7 @@ def Arith_AddIOp : Arith_IntBinaryOpWithOverflowFlags<"addi", [Commutative]> {
     these is required to be the same type. This type may be an integer scalar type, 
     a vector whose element type is integer, or a tensor of integers.
 
-    This op supports `nuw`/`nsw` overflow flags which stands stand for
+    This op supports `nuw`/`nsw` overflow flags which stands for
     "No Unsigned Wrap" and "No Signed Wrap", respectively. If the `nuw` and/or
     `nsw` flags are present, and an unsigned/signed overflow occurs
     (respectively), the result is poison.
@@ -321,7 +321,7 @@ def Arith_SubIOp : Arith_IntBinaryOpWithOverflowFlags<"subi"> {
     these is required to be the same type. This type may be an integer scalar type,
     a vector whose element type is integer, or a tensor of integers.
 
-    This op supports `nuw`/`nsw` overflow flags which stands stand for
+    This op supports `nuw`/`nsw` overflow flags which stands for
     "No Unsigned Wrap" and "No Signed Wrap", respectively. If the `nuw` and/or
     `nsw` flags are present, and an unsigned/signed overflow occurs
     (respectively), the result is poison.
@@ -367,7 +367,7 @@ def Arith_MulIOp : Arith_IntBinaryOpWithOverflowFlags<"muli",
     these is required to be the same type. This type may be an integer scalar type,
     a vector whose element type is integer, or a tensor of integers.
 
-    This op supports `nuw`/`nsw` overflow flags which stands stand for
+    This op supports `nuw`/`nsw` overflow flags which stands for
     "No Unsigned Wrap" and "No Signed Wrap", respectively. If the `nuw` and/or
     `nsw` flags are present, and an unsigned/signed overflow occurs
     (respectively), the result is poison.
@@ -800,7 +800,7 @@ def Arith_ShLIOp : Arith_IntBinaryOpWithOverflowFlags<"shli"> {
     operand is greater or equal than the bitwidth of the first operand, then the
     operation returns poison.
 
-    This op supports `nuw`/`nsw` overflow flags which stands stand for
+    This op supports `nuw`/`nsw` overflow flags which stands for
     "No Unsigned Wrap" and "No Signed Wrap", respectively. If the `nuw` and/or
     `nsw` flags are present, and an unsigned/signed overflow occurs
     (respectively), the result is poison.
@@ -1271,7 +1271,11 @@ def Arith_ScalingExtFOp
 // TruncIOp
 //===----------------------------------------------------------------------===//
 
-def Arith_TruncIOp : Arith_IToICastOp<"trunci"> {
+def Arith_TruncIOp : Op<Arith_Dialect, "trunci",
+    [Pure, SameOperandsAndResultShape, SameInputOutputTensorDims,
+     DeclareOpInterfaceMethods<CastOpInterface>,
+     DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
+     DeclareOpInterfaceMethods<ArithIntegerOverflowFlagsInterface>]> {
   let summary = "integer truncation operation";
   let description = [{
     The integer truncation operation takes an integer input of
@@ -1279,17 +1283,37 @@ def Arith_TruncIOp : Arith_IToICastOp<"trunci"> {
     bit-width must be smaller than the input bit-width (N < M).
     The top-most (N - M) bits of the input are discarded.
 
+    This op supports `nuw`/`nsw` overflow flags which stands for "No Unsigned
+    Wrap" and "No Signed Wrap", respectively. If the nuw keyword is present,
+    and any of the truncated bits are non-zero, the result is a poison value.
+    If the nsw keyword is present, and any of the truncated bits are not the
+    same as the top bit of the truncation result, the result is a poison value.
+
     Example:
 
     ```mlir
+      // Scalar truncation.
       %1 = arith.constant 21 : i5     // %1 is 0b10101
       %2 = arith.trunci %1 : i5 to i4 // %2 is 0b0101
       %3 = arith.trunci %1 : i5 to i3 // %3 is 0b101
 
-      %5 = arith.trunci %0 : vector<2 x i32> to vector<2 x i16>
+      // Vector truncation.
+      %4 = arith.trunci %0 : vector<2 x i32> to vector<2 x i16>
+
+      // Scalar truncation with overflow flags.
+      %5 = arith.trunci %a overflow<nsw, nuw> : i32 to i16
     ```
   }];
 
+  let arguments = (ins
+      SignlessFixedWidthIntegerLike:$in,
+      DefaultValuedAttr<Arith_IntegerOverflowAttr,
+          "::mlir::arith::IntegerOverflowFlags::none">:$overflowFlags);
+  let results = (outs SignlessFixedWidthIntegerLike:$out);
+  let assemblyFormat = [{
+    $in (`overflow` `` $overflowFlags^)? attr-dict
+    `:` type($in) `to` type($out)
+  }];
   let hasFolder = 1;
   let hasCanonicalizer = 1;
   let hasVerifier = 1;
diff --git a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
index ced18a48766b..b8e5aa87244f 100644
--- a/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
+++ b/mlir/lib/Conversion/ArithToLLVM/ArithToLLVM.cpp
@@ -163,7 +163,8 @@ using ConstrainedTruncFOpLowering = ConstrainedVectorConvertToLLVMPattern<
     arith::TruncFOp, LLVM::ConstrainedFPTruncIntr, true,
     arith::AttrConverterConstrainedFPToLLVM>;
 using TruncIOpLowering =
-    VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp>;
+    VectorConvertToLLVMPattern<arith::TruncIOp, LLVM::TruncOp,
+                               arith::AttrConvertOverflowToLLVM>;
 using UIToFPOpLowering =
     VectorConvertToLLVMPattern<arith::UIToFPOp, LLVM::UIToFPOp>;
 using XOrIOpLowering = VectorConvertToLLVMPattern<arith::XOrIOp, LLVM::XOrOp>;
diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index 13eb97a910bd..b61612436eb7 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -378,14 +378,14 @@ def TruncationMatchesShiftAmount :
 
 // trunci(extsi(x)) -> extsi(x), when only the sign-extension bits are truncated
 def TruncIExtSIToExtSI :
-    Pat<(Arith_TruncIOp:$tr (Arith_ExtSIOp:$ext $x)),
+    Pat<(Arith_TruncIOp:$tr (Arith_ExtSIOp:$ext $x), $overflow),
         (Arith_ExtSIOp $x),
         [(ValueWiderThan $ext, $tr),
          (ValueWiderThan $tr, $x)]>;
 
 // trunci(extui(x)) -> extui(x), when only the zero-extension bits are truncated
 def TruncIExtUIToExtUI :
-    Pat<(Arith_TruncIOp:$tr (Arith_ExtUIOp:$ext $x)),
+    Pat<(Arith_TruncIOp:$tr (Arith_ExtUIOp:$ext $x), $overflow),
         (Arith_ExtUIOp $x),
         [(ValueWiderThan $ext, $tr),
          (ValueWiderThan $tr, $x)]>;
@@ -393,8 +393,8 @@ def TruncIExtUIToExtUI :
 // trunci(shrsi(x, c)) -> trunci(shrui(x, c))
 def TruncIShrSIToTrunciShrUI :
     Pat<(Arith_TruncIOp:$tr
-          (Arith_ShRSIOp $x, (ConstantLikeMatcher TypedAttrInterface:$c0))),
-        (Arith_TruncIOp (Arith_ShRUIOp $x, (Arith_ConstantOp (cast<"TypedAttr"> $c0)))),
+          (Arith_ShRSIOp $x, (ConstantLikeMatcher TypedAttrInterface:$c0)), $overflow),
+        (Arith_TruncIOp (Arith_ShRUIOp $x, (Arith_ConstantOp (cast<"TypedAttr"> $c0))), $overflow),
         [(TruncationMatchesShiftAmount $x, $tr, $c0)]>;
 
 // trunci(shrui(mul(sext(x), sext(y)), c)) -> mulsi_extended(x, y)
@@ -402,7 +402,7 @@ def TruncIShrUIMulIToMulSIExtended :
     Pat<(Arith_TruncIOp:$tr (Arith_ShRUIOp
                               (Arith_MulIOp:$mul
                                 (Arith_ExtSIOp $x), (Arith_ExtSIOp $y), $ovf1),
-                              (ConstantLikeMatcher AnyAttr:$c0))),
+                              (ConstantLikeMatcher AnyAttr:$c0)), $overflow),
         (Arith_MulSIExtendedOp:$res__1 $x, $y),
       [(ValuesWithSameType $tr, $x, $y),
        (ValueWiderThan $mul, $x),
@@ -413,7 +413,7 @@ def TruncIShrUIMulIToMulUIExtended :
     Pat<(Arith_TruncIOp:$tr (Arith_ShRUIOp
                               (Arith_MulIOp:$mul
                                 (Arith_ExtUIOp $x), (Arith_ExtUIOp $y), $ovf1),
-                              (ConstantLikeMatcher AnyAttr:$c0))),
+                              (ConstantLikeMatcher AnyAttr:$c0)), $overflow),
         (Arith_MulUIExtendedOp:$res__1 $x, $y),
       [(ValuesWithSameType $tr, $x, $y),
        (ValueWiderThan $mul, $x),
diff --git a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
index e0d974ea7404..83bdbe1f6711 100644
--- a/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
+++ b/mlir/test/Conversion/ArithToLLVM/arith-to-llvm.mlir
@@ -731,6 +731,8 @@ func.func @ops_supporting_overflow(%arg0: i64, %arg1: i64) {
   %2 = arith.muli %arg0, %arg1 overflow<nsw, nuw> : i64
   // CHECK: %{{.*}} = llvm.shl %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
   %3 = arith.shli %arg0, %arg1 overflow<nsw, nuw> : i64
+  // CHECK: %{{.*}} = llvm.trunc %{{.*}} overflow<nsw, nuw> : i64 to i32
+  %4 = arith.trunci %arg0 overflow<nsw, nuw> : i64 to i32
   return
 }
 
diff --git a/mlir/test/Dialect/Arith/ops.mlir b/mlir/test/Dialect/Arith/ops.mlir
index f684e02344a5..1e656e84da83 100644
--- a/mlir/test/Dialect/Arith/ops.mlir
+++ b/mlir/test/Dialect/Arith/ops.mlir
@@ -1159,5 +1159,7 @@ func.func @intflags_func(%arg0: i64, %arg1: i64) {
   %2 = arith.muli %arg0, %arg1 overflow<nsw, nuw> : i64
   // CHECK: %{{.*}} = arith.shli %{{.*}}, %{{.*}} overflow<nsw, nuw> : i64
   %3 = arith.shli %arg0, %arg1 overflow<nsw, nuw> : i64
+  // CHECK: %{{.*}} = arith.trunci %{{.*}} overflow<nsw, nuw> : i64 to i32
+  %4 = arith.trunci %arg0 overflow<nsw, nuw> : i64 to i32
   return
 }
-- 
cgit v1.2.3


From b73720cf6c5380854bf27d4453abf21cc87ae642 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Don=C3=A1t=20Nagy?= <donat.nagy@ericsson.com>
Date: Thu, 19 Jun 2025 14:00:36 +0200
Subject: [analyzer] Conversion to CheckerFamily: DynamicTypePropagation
 (#144735)

This commit converts the class DynamicTypePropagation to a very simple
checker family, which has only one checker frontend -- but also supports
enabling the backend ("modeling checker") without the frontend.

As a tangentially related change, this commit adds the backend of
DynamicTypePropagation as a dependency of alpha.core.DynamicTypeChecker
in Checkers.td, because the header comment of DynamicTypeChecker.cpp
claims that it depends on DynamicTypePropagation and the source code
seems to confirm this.

(The lack of this dependency relationship didn't cause problems, because
'core.DynamicTypePropagation' is in the group 'core', so it is
practically always active. However, explicitly declaring the dependency
clarifies the fact that the separate existence of the modeling checker
is warranted.)
---
 .../clang/StaticAnalyzer/Checkers/Checkers.td      | 10 ++--
 .../Checkers/DynamicTypePropagation.cpp            | 55 ++++++++++------------
 2 files changed, 32 insertions(+), 33 deletions(-)

diff --git a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
index 211ce585fbac..2234143004b6 100644
--- a/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
+++ b/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td
@@ -294,10 +294,12 @@ def TestAfterDivZeroChecker : Checker<"TestAfterDivZero">,
            "Either the comparison is useless or there is division by zero.">,
   Documentation<HasDocumentation>;
 
-def DynamicTypeChecker : Checker<"DynamicTypeChecker">,
-  HelpText<"Check for cases where the dynamic and the static type of an object "
-           "are unrelated.">,
-  Documentation<HasDocumentation>;
+def DynamicTypeChecker
+    : Checker<"DynamicTypeChecker">,
+      HelpText<"Check for cases where the dynamic and the static type of an "
+               "object are unrelated.">,
+      Dependencies<[DynamicTypePropagation]>,
+      Documentation<HasDocumentation>;
 
 def StackAddrAsyncEscapeChecker : Checker<"StackAddressAsyncEscape">,
   HelpText<"Check that addresses to stack memory do not escape the function">,
diff --git a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index 344be0b176c5..4982cd59b0a4 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -49,15 +49,19 @@ REGISTER_MAP_WITH_PROGRAMSTATE(MostSpecializedTypeArgsMap, SymbolRef,
                                const ObjCObjectPointerType *)
 
 namespace {
-class DynamicTypePropagation:
-    public Checker< check::PreCall,
-                    check::PostCall,
-                    check::DeadSymbols,
-                    check::PostStmt<CastExpr>,
-                    check::PostStmt<CXXNewExpr>,
-                    check::PreObjCMessage,
-                    check::PostObjCMessage > {
+class DynamicTypePropagation
+    : public CheckerFamily<check::PreCall, check::PostCall, check::DeadSymbols,
+                           check::PostStmt<CastExpr>,
+                           check::PostStmt<CXXNewExpr>, check::PreObjCMessage,
+                           check::PostObjCMessage> {
+public:
+  // This checker family implements only one frontend, but -- unlike a simple
+  // Checker -- its backend can be enabled (by the checker DynamicTypeChecker
+  // which depends on it) without enabling the frontend.
+  CheckerFrontendWithBugType ObjCGenericsChecker{
+      "Generics", categories::CoreFoundationObjectiveC};
 
+private:
   /// Return a better dynamic type if one can be derived from the cast.
   const ObjCObjectPointerType *getBetterObjCType(const Expr *CastE,
                                                  CheckerContext &C) const;
@@ -66,13 +70,6 @@ class DynamicTypePropagation:
                                               ProgramStateRef &State,
                                               CheckerContext &C) const;
 
-  mutable std::unique_ptr<BugType> ObjCGenericsBugType;
-  void initBugType() const {
-    if (!ObjCGenericsBugType)
-      ObjCGenericsBugType.reset(new BugType(
-          GenericCheckName, "Generics", categories::CoreFoundationObjectiveC));
-  }
-
   class GenericsBugVisitor : public BugReporterVisitor {
   public:
     GenericsBugVisitor(SymbolRef S) : Sym(S) {}
@@ -106,9 +103,8 @@ public:
   void checkPreObjCMessage(const ObjCMethodCall &M, CheckerContext &C) const;
   void checkPostObjCMessage(const ObjCMethodCall &M, CheckerContext &C) const;
 
-  /// This value is set to true, when the Generics checker is turned on.
-  bool CheckGenerics = false;
-  CheckerNameRef GenericCheckName;
+  /// Identifies this checker family for debugging purposes.
+  StringRef getDebugTag() const override { return "DynamicTypePropagation"; }
 };
 
 bool isObjCClassType(QualType Type) {
@@ -1026,10 +1022,9 @@ void DynamicTypePropagation::reportGenericsBug(
     const ObjCObjectPointerType *From, const ObjCObjectPointerType *To,
     ExplodedNode *N, SymbolRef Sym, CheckerContext &C,
     const Stmt *ReportedNode) const {
-  if (!CheckGenerics)
+  if (!ObjCGenericsChecker.isEnabled())
     return;
 
-  initBugType();
   SmallString<192> Buf;
   llvm::raw_svector_ostream OS(Buf);
   OS << "Conversion from value of type '";
@@ -1037,7 +1032,7 @@ void DynamicTypePropagation::reportGenericsBug(
   OS << "' to incompatible type '";
   QualType::print(To, Qualifiers(), OS, C.getLangOpts(), llvm::Twine());
   OS << "'";
-  auto R = std::make_unique<PathSensitiveBugReport>(*ObjCGenericsBugType,
+  auto R = std::make_unique<PathSensitiveBugReport>(ObjCGenericsChecker,
                                                     OS.str(), N);
   R->markInteresting(Sym);
   R->addVisitor(std::make_unique<GenericsBugVisitor>(Sym));
@@ -1102,20 +1097,22 @@ PathDiagnosticPieceRef DynamicTypePropagation::GenericsBugVisitor::VisitNode(
 }
 
 /// Register checkers.
-void ento::registerObjCGenericsChecker(CheckerManager &mgr) {
-  DynamicTypePropagation *checker = mgr.getChecker<DynamicTypePropagation>();
-  checker->CheckGenerics = true;
-  checker->GenericCheckName = mgr.getCurrentCheckerName();
+void ento::registerObjCGenericsChecker(CheckerManager &Mgr) {
+  Mgr.getChecker<DynamicTypePropagation>()->ObjCGenericsChecker.enable(Mgr);
 }
 
-bool ento::shouldRegisterObjCGenericsChecker(const CheckerManager &mgr) {
+bool ento::shouldRegisterObjCGenericsChecker(const CheckerManager &) {
   return true;
 }
 
-void ento::registerDynamicTypePropagation(CheckerManager &mgr) {
-  mgr.registerChecker<DynamicTypePropagation>();
+void ento::registerDynamicTypePropagation(CheckerManager &Mgr) {
+  // The checker 'core.DynamicTypeChecker' relies on the modeling implemented
+  // in the class 'DynamicTypePropagation', so this "modeling checker" can
+  // register the 'DynamicTypePropagation' backend for its callbacks without
+  // enabling its frontend.
+  Mgr.getChecker<DynamicTypePropagation>();
 }
 
-bool ento::shouldRegisterDynamicTypePropagation(const CheckerManager &mgr) {
+bool ento::shouldRegisterDynamicTypePropagation(const CheckerManager &) {
   return true;
 }
-- 
cgit v1.2.3


From 2b4d757290226e0185e17294339aae1588efd07e Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko <atrosinenko@accesssoftek.com>
Date: Thu, 19 Jun 2025 15:15:26 +0300
Subject: [BOLT] Gadget scanner: detect authentication oracles (#135663)

Implement the detection of authentication instructions whose results can
be inspected by an attacker to know whether authentication succeeded.

As the properties of output registers of authentication instructions are
inspected, add a second set of analysis-related classes to iterate over
the instructions in reverse order.
---
 bolt/include/bolt/Passes/PAuthGadgetScanner.h      |  25 +-
 bolt/lib/Passes/PAuthGadgetScanner.cpp             | 652 ++++++++++++++++-
 .../AArch64/gs-pauth-authentication-oracles.s      | 812 +++++++++++++++++++++
 .../AArch64/gs-pauth-debug-output.s                |  84 ++-
 4 files changed, 1534 insertions(+), 39 deletions(-)
 create mode 100644 bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s

diff --git a/bolt/include/bolt/Passes/PAuthGadgetScanner.h b/bolt/include/bolt/Passes/PAuthGadgetScanner.h
index c6b9cc2eb4b9..721fd664a325 100644
--- a/bolt/include/bolt/Passes/PAuthGadgetScanner.h
+++ b/bolt/include/bolt/Passes/PAuthGadgetScanner.h
@@ -199,8 +199,7 @@ namespace PAuthGadgetScanner {
 // to distinguish intermediate and final results at the type level.
 //
 // Here is an overview of issue life-cycle:
-// * an analysis (SrcSafetyAnalysis at now, DstSafetyAnalysis will be added
-//   later to support the detection of authentication oracles) computes register
+// * an analysis (SrcSafetyAnalysis or DstSafetyAnalysis) computes register
 //   state for each instruction in the function.
 // * for each instruction, it is checked whether it is a gadget of some kind,
 //   taking the computed state into account. If a gadget is found, its kind
@@ -273,6 +272,11 @@ public:
   virtual ~ExtraInfo() {}
 };
 
+/// The set of instructions writing to the affected register in an unsafe
+/// manner.
+///
+/// This is a hint to be printed alongside the report. It should be further
+/// analyzed by the user.
 class ClobberingInfo : public ExtraInfo {
   SmallVector<MCInstReference> ClobberingInstrs;
 
@@ -282,6 +286,20 @@ public:
   void print(raw_ostream &OS, const MCInstReference Location) const override;
 };
 
+/// The set of instructions leaking the authenticated pointer before the
+/// result of authentication was checked.
+///
+/// This is a hint to be printed alongside the report. It should be further
+/// analyzed by the user.
+class LeakageInfo : public ExtraInfo {
+  SmallVector<MCInstReference> LeakingInstrs;
+
+public:
+  LeakageInfo(ArrayRef<MCInstReference> Instrs) : LeakingInstrs(Instrs) {}
+
+  void print(raw_ostream &OS, const MCInstReference Location) const override;
+};
+
 /// A brief version of a report that can be further augmented with the details.
 ///
 /// A half-baked report produced on the first run of the analysis. An extra,
@@ -322,6 +340,9 @@ class FunctionAnalysisContext {
   void findUnsafeUses(SmallVector<PartialReport<MCPhysReg>> &Reports);
   void augmentUnsafeUseReports(ArrayRef<PartialReport<MCPhysReg>> Reports);
 
+  void findUnsafeDefs(SmallVector<PartialReport<MCPhysReg>> &Reports);
+  void augmentUnsafeDefReports(ArrayRef<PartialReport<MCPhysReg>> Reports);
+
   /// Process the reports which do not have to be augmented, and remove them
   /// from Reports.
   void handleSimpleReports(SmallVector<PartialReport<MCPhysReg>> &Reports);
diff --git a/bolt/lib/Passes/PAuthGadgetScanner.cpp b/bolt/lib/Passes/PAuthGadgetScanner.cpp
index 971ea5fdef42..7682d7fe2c54 100644
--- a/bolt/lib/Passes/PAuthGadgetScanner.cpp
+++ b/bolt/lib/Passes/PAuthGadgetScanner.cpp
@@ -152,6 +152,8 @@ public:
 //    in the gadgets to be reported. This information is used in the second run
 //    to also track which instructions last wrote to those registers.
 
+typedef SmallPtrSet<const MCInst *, 4> SetOfRelatedInsts;
+
 /// A state representing which registers are safe to use by an instruction
 /// at a given program point.
 ///
@@ -195,7 +197,7 @@ struct SrcState {
   /// pac-ret analysis, the expectation is that almost all return instructions
   /// only use register `X30`, and therefore, this vector will probably have
   /// length 1 in the second run.
-  std::vector<SmallPtrSet<const MCInst *, 4>> LastInstWritingReg;
+  std::vector<SetOfRelatedInsts> LastInstWritingReg;
 
   /// Construct an empty state.
   SrcState() {}
@@ -230,12 +232,11 @@ struct SrcState {
   bool operator!=(const SrcState &RHS) const { return !((*this) == RHS); }
 };
 
-static void
-printLastInsts(raw_ostream &OS,
-               ArrayRef<SmallPtrSet<const MCInst *, 4>> LastInstWritingReg) {
+static void printInstsShort(raw_ostream &OS,
+                            ArrayRef<SetOfRelatedInsts> Insts) {
   OS << "Insts: ";
-  for (unsigned I = 0; I < LastInstWritingReg.size(); ++I) {
-    auto &Set = LastInstWritingReg[I];
+  for (unsigned I = 0; I < Insts.size(); ++I) {
+    auto &Set = Insts[I];
     OS << "[" << I << "](";
     for (const MCInst *MCInstP : Set)
       OS << MCInstP << " ";
@@ -243,14 +244,14 @@ printLastInsts(raw_ostream &OS,
   }
 }
 
-raw_ostream &operator<<(raw_ostream &OS, const SrcState &S) {
+static raw_ostream &operator<<(raw_ostream &OS, const SrcState &S) {
   OS << "src-state<";
   if (S.empty()) {
     OS << "empty";
   } else {
     OS << "SafeToDerefRegs: " << S.SafeToDerefRegs << ", ";
     OS << "TrustedRegs: " << S.TrustedRegs << ", ";
-    printLastInsts(OS, S.LastInstWritingReg);
+    printInstsShort(OS, S.LastInstWritingReg);
   }
   OS << ">";
   return OS;
@@ -279,7 +280,7 @@ void SrcStatePrinter::print(raw_ostream &OS, const SrcState &S) const {
     OS << ", TrustedRegs: ";
     RegStatePrinter.print(OS, S.TrustedRegs);
     OS << ", ";
-    printLastInsts(OS, S.LastInstWritingReg);
+    printInstsShort(OS, S.LastInstWritingReg);
   }
   OS << ">";
 }
@@ -323,13 +324,12 @@ protected:
   DenseMap<const MCInst *, std::pair<MCPhysReg, const MCInst *>>
       CheckerSequenceInfo;
 
-  SmallPtrSet<const MCInst *, 4> &lastWritingInsts(SrcState &S,
-                                                   MCPhysReg Reg) const {
+  SetOfRelatedInsts &lastWritingInsts(SrcState &S, MCPhysReg Reg) const {
     unsigned Index = RegsToTrackInstsFor.getIndex(Reg);
     return S.LastInstWritingReg[Index];
   }
-  const SmallPtrSet<const MCInst *, 4> &lastWritingInsts(const SrcState &S,
-                                                         MCPhysReg Reg) const {
+  const SetOfRelatedInsts &lastWritingInsts(const SrcState &S,
+                                            MCPhysReg Reg) const {
     unsigned Index = RegsToTrackInstsFor.getIndex(Reg);
     return S.LastInstWritingReg[Index];
   }
@@ -433,8 +433,7 @@ protected:
     SrcStatePrinter P(BC);
     LLVM_DEBUG({
       dbgs() << "  SrcSafetyAnalysis::ComputeNext(";
-      BC.InstPrinter->printInst(&const_cast<MCInst &>(Point), 0, "", *BC.STI,
-                                dbgs());
+      BC.InstPrinter->printInst(&Point, 0, "", *BC.STI, dbgs());
       dbgs() << ", ";
       P.print(dbgs(), Cur);
       dbgs() << ")\n";
@@ -612,6 +611,42 @@ protected:
   StringRef getAnnotationName() const { return "DataflowSrcSafetyAnalysis"; }
 };
 
+/// A helper base class for implementing a simplified counterpart of a dataflow
+/// analysis for functions without CFG information.
+template <typename StateTy> class CFGUnawareAnalysis {
+  BinaryContext &BC;
+  BinaryFunction &BF;
+  MCPlusBuilder::AllocatorIdTy AllocId;
+  unsigned StateAnnotationIndex;
+
+  void cleanStateAnnotations() {
+    for (auto &I : BF.instrs())
+      BC.MIB->removeAnnotation(I.second, StateAnnotationIndex);
+  }
+
+protected:
+  CFGUnawareAnalysis(BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId,
+                     StringRef AnnotationName)
+      : BC(BF.getBinaryContext()), BF(BF), AllocId(AllocId) {
+    StateAnnotationIndex = BC.MIB->getOrCreateAnnotationIndex(AnnotationName);
+  }
+
+  void setState(MCInst &Inst, const StateTy &S) {
+    // Check if we need to remove an old annotation (this is the case if
+    // this is the second, detailed run of the analysis).
+    if (BC.MIB->hasAnnotation(Inst, StateAnnotationIndex))
+      BC.MIB->removeAnnotation(Inst, StateAnnotationIndex);
+    // Attach the state.
+    BC.MIB->addAnnotation(Inst, StateAnnotationIndex, S, AllocId);
+  }
+
+  const StateTy &getState(const MCInst &Inst) const {
+    return BC.MIB->getAnnotationAs<StateTy>(Inst, StateAnnotationIndex);
+  }
+
+  virtual ~CFGUnawareAnalysis() { cleanStateAnnotations(); }
+};
+
 // A simplified implementation of DataflowSrcSafetyAnalysis for functions
 // lacking CFG information.
 //
@@ -646,15 +681,10 @@ protected:
 // of instructions without labels in between. These sequences can be processed
 // the same way basic blocks are processed by data-flow analysis, assuming
 // pessimistically that all registers are unsafe at the start of each sequence.
-class CFGUnawareSrcSafetyAnalysis : public SrcSafetyAnalysis {
+class CFGUnawareSrcSafetyAnalysis : public SrcSafetyAnalysis,
+                                    public CFGUnawareAnalysis<SrcState> {
+  using SrcSafetyAnalysis::BC;
   BinaryFunction &BF;
-  MCPlusBuilder::AllocatorIdTy AllocId;
-  unsigned StateAnnotationIndex;
-
-  void cleanStateAnnotations() {
-    for (auto &I : BF.instrs())
-      BC.MIB->removeAnnotation(I.second, StateAnnotationIndex);
-  }
 
   /// Creates a state with all registers marked unsafe (not to be confused
   /// with empty state).
@@ -666,9 +696,8 @@ public:
   CFGUnawareSrcSafetyAnalysis(BinaryFunction &BF,
                               MCPlusBuilder::AllocatorIdTy AllocId,
                               ArrayRef<MCPhysReg> RegsToTrackInstsFor)
-      : SrcSafetyAnalysis(BF, RegsToTrackInstsFor), BF(BF), AllocId(AllocId) {
-    StateAnnotationIndex =
-        BC.MIB->getOrCreateAnnotationIndex("CFGUnawareSrcSafetyAnalysis");
+      : SrcSafetyAnalysis(BF, RegsToTrackInstsFor),
+        CFGUnawareAnalysis(BF, AllocId, "CFGUnawareSrcSafetyAnalysis"), BF(BF) {
   }
 
   void run() override {
@@ -687,12 +716,8 @@ public:
         S = createUnsafeState();
       }
 
-      // Check if we need to remove an old annotation (this is the case if
-      // this is the second, detailed, run of the analysis).
-      if (BC.MIB->hasAnnotation(Inst, StateAnnotationIndex))
-        BC.MIB->removeAnnotation(Inst, StateAnnotationIndex);
       // Attach the state *before* this instruction executes.
-      BC.MIB->addAnnotation(Inst, StateAnnotationIndex, S, AllocId);
+      setState(Inst, S);
 
       // Compute the state after this instruction executes.
       S = computeNext(Inst, S);
@@ -700,10 +725,8 @@ public:
   }
 
   const SrcState &getStateBefore(const MCInst &Inst) const override {
-    return BC.MIB->getAnnotationAs<SrcState>(Inst, StateAnnotationIndex);
+    return getState(Inst);
   }
-
-  ~CFGUnawareSrcSafetyAnalysis() { cleanStateAnnotations(); }
 };
 
 std::shared_ptr<SrcSafetyAnalysis>
@@ -717,6 +740,478 @@ SrcSafetyAnalysis::create(BinaryFunction &BF,
                                                        RegsToTrackInstsFor);
 }
 
+/// A state representing which registers are safe to be used as the destination
+/// operand of an authentication instruction.
+///
+/// Similar to SrcState, it is the responsibility of the analysis to take
+/// register aliasing into account.
+///
+/// Depending on the implementation (such as whether FEAT_FPAC is implemented
+/// by an AArch64 CPU or not), it may be possible that an authentication
+/// instruction returns an invalid pointer on failure instead of terminating
+/// the program immediately (assuming the program will crash as soon as that
+/// pointer is dereferenced). Since few bits are usually allocated for the PAC
+/// field (such as less than 16 bits on a typical AArch64 system), an attacker
+/// can try every possible signature and guess the correct one if there is a
+/// gadget that tells whether the particular pointer has a correct signature
+/// (a so called "authentication oracle"). For that reason, it should be
+/// impossible for an attacker to test if a pointer is correctly signed -
+/// either the program should be terminated on authentication failure or
+/// the result of authentication should not be accessible to an attacker.
+///
+/// Considering the instructions in forward order as they are executed, a
+/// restricted set of operations can be allowed on any register containing a
+/// value derived from the result of an authentication instruction until that
+/// value is checked not to contain the result of a failed authentication.
+/// In DstSafetyAnalysis, these rules are adapted, so that the safety property
+/// for a register is computed by iterating the instructions in backward order.
+/// Then the resulting properties are used at authentication instruction sites
+/// to check output registers and report the particular instruction if it writes
+/// to an unsafe register.
+///
+/// Another approach would be to simulate the above rules as-is, iterating over
+/// the instructions in forward direction. To make it possible to report the
+/// particular instructions as oracles, this would probably require tracking
+/// references to these instructions for each register currently containing
+/// sensitive data.
+///
+/// In DstSafetyAnalysis, the source register Xn of an instruction Inst is safe
+/// if at least one of the following is true:
+/// * Inst checks if Xn contains the result of a successful authentication and
+///   terminates the program on failure. Note that Inst can either naturally
+///   dereference Xn (load, branch, return, etc. instructions) or be the first
+///   instruction of an explicit checking sequence.
+/// * Inst performs safe address arithmetic AND both source and result
+///   registers, as well as any temporary registers, must be safe after
+///   execution of Inst (temporaries are not used on AArch64 and thus not
+///   currently supported/allowed).
+///   See MCPlusBuilder::analyzeAddressArithmeticsForPtrAuth for the details.
+/// * Inst fully overwrites Xn with a constant.
+struct DstState {
+  /// The set of registers whose values cannot be inspected by an attacker in
+  /// a way usable as an authentication oracle. The results of authentication
+  /// instructions should only be written to such registers.
+  BitVector CannotEscapeUnchecked;
+
+  /// A vector of sets, only used on the second analysis run.
+  /// Each element in this vector represents one of the tracked registers.
+  /// For each such register we track the set of first instructions that leak
+  /// the authenticated pointer before it was checked. This is intended to
+  /// provide clues on which instruction made the particular register unsafe.
+  ///
+  /// Please note that the mapping from MCPhysReg values to indexes in this
+  /// vector is provided by RegsToTrackInstsFor field of DstSafetyAnalysis.
+  std::vector<SetOfRelatedInsts> FirstInstLeakingReg;
+
+  /// Constructs an empty state.
+  DstState() {}
+
+  DstState(unsigned NumRegs, unsigned NumRegsToTrack)
+      : CannotEscapeUnchecked(NumRegs), FirstInstLeakingReg(NumRegsToTrack) {}
+
+  DstState &merge(const DstState &StateIn) {
+    if (StateIn.empty())
+      return *this;
+    if (empty())
+      return (*this = StateIn);
+
+    CannotEscapeUnchecked &= StateIn.CannotEscapeUnchecked;
+    for (unsigned I = 0; I < FirstInstLeakingReg.size(); ++I)
+      for (const MCInst *J : StateIn.FirstInstLeakingReg[I])
+        FirstInstLeakingReg[I].insert(J);
+    return *this;
+  }
+
+  /// Returns true if this object does not store state of any registers -
+  /// neither safe, nor unsafe ones.
+  bool empty() const { return CannotEscapeUnchecked.empty(); }
+
+  bool operator==(const DstState &RHS) const {
+    return CannotEscapeUnchecked == RHS.CannotEscapeUnchecked &&
+           FirstInstLeakingReg == RHS.FirstInstLeakingReg;
+  }
+  bool operator!=(const DstState &RHS) const { return !((*this) == RHS); }
+};
+
+static raw_ostream &operator<<(raw_ostream &OS, const DstState &S) {
+  OS << "dst-state<";
+  if (S.empty()) {
+    OS << "empty";
+  } else {
+    OS << "CannotEscapeUnchecked: " << S.CannotEscapeUnchecked << ", ";
+    printInstsShort(OS, S.FirstInstLeakingReg);
+  }
+  OS << ">";
+  return OS;
+}
+
+class DstStatePrinter {
+public:
+  void print(raw_ostream &OS, const DstState &S) const;
+  explicit DstStatePrinter(const BinaryContext &BC) : BC(BC) {}
+
+private:
+  const BinaryContext &BC;
+};
+
+void DstStatePrinter::print(raw_ostream &OS, const DstState &S) const {
+  RegStatePrinter RegStatePrinter(BC);
+  OS << "dst-state<";
+  if (S.empty()) {
+    assert(S.CannotEscapeUnchecked.empty());
+    assert(S.FirstInstLeakingReg.empty());
+    OS << "empty";
+  } else {
+    OS << "CannotEscapeUnchecked: ";
+    RegStatePrinter.print(OS, S.CannotEscapeUnchecked);
+    OS << ", ";
+    printInstsShort(OS, S.FirstInstLeakingReg);
+  }
+  OS << ">";
+}
+
+/// Computes which registers are safe to be written to by auth instructions.
+///
+/// This is the base class for two implementations: a dataflow-based analysis
+/// which is intended to be used for most functions and a simplified CFG-unaware
+/// version for functions without reconstructed CFG.
+class DstSafetyAnalysis {
+public:
+  DstSafetyAnalysis(BinaryFunction &BF, ArrayRef<MCPhysReg> RegsToTrackInstsFor)
+      : BC(BF.getBinaryContext()), NumRegs(BC.MRI->getNumRegs()),
+        RegsToTrackInstsFor(RegsToTrackInstsFor) {}
+
+  virtual ~DstSafetyAnalysis() {}
+
+  static std::shared_ptr<DstSafetyAnalysis>
+  create(BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId,
+         ArrayRef<MCPhysReg> RegsToTrackInstsFor);
+
+  virtual void run() = 0;
+  virtual const DstState &getStateAfter(const MCInst &Inst) const = 0;
+
+protected:
+  BinaryContext &BC;
+  const unsigned NumRegs;
+
+  const TrackedRegisters RegsToTrackInstsFor;
+
+  /// Stores information about the detected instruction sequences emitted to
+  /// check an authenticated pointer. Specifically, if such sequence is detected
+  /// in a basic block, it maps the first instruction of that sequence to the
+  /// register being checked.
+  ///
+  /// As the detection of such sequences requires iterating over the adjacent
+  /// instructions, it should be done before calling computeNext(), which
+  /// operates on separate instructions.
+  DenseMap<const MCInst *, MCPhysReg> RegCheckedAt;
+
+  SetOfRelatedInsts &firstLeakingInsts(DstState &S, MCPhysReg Reg) const {
+    unsigned Index = RegsToTrackInstsFor.getIndex(Reg);
+    return S.FirstInstLeakingReg[Index];
+  }
+  const SetOfRelatedInsts &firstLeakingInsts(const DstState &S,
+                                             MCPhysReg Reg) const {
+    unsigned Index = RegsToTrackInstsFor.getIndex(Reg);
+    return S.FirstInstLeakingReg[Index];
+  }
+
+  /// Creates a state with all registers marked unsafe (not to be confused
+  /// with empty state).
+  DstState createUnsafeState() {
+    return DstState(NumRegs, RegsToTrackInstsFor.getNumTrackedRegisters());
+  }
+
+  /// Returns the set of registers that can be leaked by this instruction.
+  /// A register is considered leaked if it has any intersection with any
+  /// register read by Inst. This is similar to how the set of clobbered
+  /// registers is computed, but taking input operands instead of outputs.
+  BitVector getLeakedRegs(const MCInst &Inst) const {
+    BitVector Leaked(NumRegs);
+
+    // Assume a call can read all registers.
+    if (BC.MIB->isCall(Inst)) {
+      Leaked.set();
+      return Leaked;
+    }
+
+    // Compute the set of registers overlapping with any register used by
+    // this instruction.
+
+    const MCInstrDesc &Desc = BC.MII->get(Inst.getOpcode());
+
+    for (MCPhysReg Reg : Desc.implicit_uses())
+      Leaked |= BC.MIB->getAliases(Reg, /*OnlySmaller=*/false);
+
+    for (const MCOperand &Op : BC.MIB->useOperands(Inst)) {
+      if (Op.isReg())
+        Leaked |= BC.MIB->getAliases(Op.getReg(), /*OnlySmaller=*/false);
+    }
+
+    return Leaked;
+  }
+
+  SmallVector<MCPhysReg> getRegsMadeProtected(const MCInst &Inst,
+                                              const BitVector &LeakedRegs,
+                                              const DstState &Cur) const {
+    SmallVector<MCPhysReg> Regs;
+
+    // A pointer can be checked, or
+    if (auto CheckedReg =
+            BC.MIB->getAuthCheckedReg(Inst, /*MayOverwrite=*/true))
+      Regs.push_back(*CheckedReg);
+    if (RegCheckedAt.contains(&Inst))
+      Regs.push_back(RegCheckedAt.at(&Inst));
+
+    // ... it can be used as a branch target, or
+    if (BC.MIB->isIndirectBranch(Inst) || BC.MIB->isIndirectCall(Inst)) {
+      bool IsAuthenticated;
+      MCPhysReg BranchDestReg =
+          BC.MIB->getRegUsedAsIndirectBranchDest(Inst, IsAuthenticated);
+      assert(BranchDestReg != BC.MIB->getNoRegister());
+      if (!IsAuthenticated)
+        Regs.push_back(BranchDestReg);
+    }
+
+    // ... it can be used as a return target, or
+    if (BC.MIB->isReturn(Inst)) {
+      bool IsAuthenticated = false;
+      std::optional<MCPhysReg> RetReg =
+          BC.MIB->getRegUsedAsRetDest(Inst, IsAuthenticated);
+      if (RetReg && !IsAuthenticated)
+        Regs.push_back(*RetReg);
+    }
+
+    // ... an address can be updated in a safe manner, or
+    if (auto DstAndSrc = BC.MIB->analyzeAddressArithmeticsForPtrAuth(Inst)) {
+      MCPhysReg DstReg, SrcReg;
+      std::tie(DstReg, SrcReg) = *DstAndSrc;
+      // Note that *all* registers containing the derived values must be safe,
+      // both source and destination ones. No temporaries are supported at now.
+      if (Cur.CannotEscapeUnchecked[SrcReg] &&
+          Cur.CannotEscapeUnchecked[DstReg])
+        Regs.push_back(SrcReg);
+    }
+
+    // ... the register can be overwritten in whole with a constant: for that
+    // purpose, look for the instructions with no register inputs (neither
+    // explicit nor implicit ones) and no side effects (to rule out reading
+    // not modelled locations).
+    const MCInstrDesc &Desc = BC.MII->get(Inst.getOpcode());
+    bool HasExplicitSrcRegs = llvm::any_of(BC.MIB->useOperands(Inst),
+                                           [](auto Op) { return Op.isReg(); });
+    if (!Desc.hasUnmodeledSideEffects() && !HasExplicitSrcRegs &&
+        Desc.implicit_uses().empty()) {
+      for (const MCOperand &Def : BC.MIB->defOperands(Inst))
+        Regs.push_back(Def.getReg());
+    }
+
+    return Regs;
+  }
+
+  DstState computeNext(const MCInst &Point, const DstState &Cur) {
+    DstStatePrinter P(BC);
+    LLVM_DEBUG({
+      dbgs() << "  DstSafetyAnalysis::ComputeNext(";
+      BC.InstPrinter->printInst(&Point, 0, "", *BC.STI, dbgs());
+      dbgs() << ", ";
+      P.print(dbgs(), Cur);
+      dbgs() << ")\n";
+    });
+
+    // If this instruction is reachable by the analysis, a non-empty state will
+    // be propagated to it sooner or later. Until then, skip computeNext().
+    if (Cur.empty()) {
+      LLVM_DEBUG(
+          { dbgs() << "Skipping computeNext(Point, Cur) as Cur is empty.\n"; });
+      return DstState();
+    }
+
+    // First, compute various properties of the instruction, taking the state
+    // after its execution into account, if necessary.
+
+    BitVector LeakedRegs = getLeakedRegs(Point);
+    SmallVector<MCPhysReg> NewProtectedRegs =
+        getRegsMadeProtected(Point, LeakedRegs, Cur);
+
+    // Then, compute the state before this instruction is executed.
+    DstState Next = Cur;
+
+    Next.CannotEscapeUnchecked.reset(LeakedRegs);
+    for (MCPhysReg Reg : RegsToTrackInstsFor.getRegisters()) {
+      if (LeakedRegs[Reg])
+        firstLeakingInsts(Next, Reg) = {&Point};
+    }
+
+    BitVector NewProtectedSubregs(NumRegs);
+    for (MCPhysReg Reg : NewProtectedRegs)
+      NewProtectedSubregs |= BC.MIB->getAliases(Reg, /*OnlySmaller=*/true);
+    Next.CannotEscapeUnchecked |= NewProtectedSubregs;
+    for (MCPhysReg Reg : RegsToTrackInstsFor.getRegisters()) {
+      if (NewProtectedSubregs[Reg])
+        firstLeakingInsts(Next, Reg).clear();
+    }
+
+    LLVM_DEBUG({
+      dbgs() << "    .. result: (";
+      P.print(dbgs(), Next);
+      dbgs() << ")\n";
+    });
+
+    return Next;
+  }
+
+public:
+  std::vector<MCInstReference> getLeakingInsts(const MCInst &Inst,
+                                               BinaryFunction &BF,
+                                               MCPhysReg LeakedReg) const {
+    const DstState &S = getStateAfter(Inst);
+
+    std::vector<MCInstReference> Result;
+    for (const MCInst *Inst : firstLeakingInsts(S, LeakedReg)) {
+      MCInstReference Ref = MCInstReference::get(Inst, BF);
+      assert(Ref && "Expected Inst to be found");
+      Result.push_back(Ref);
+    }
+    return Result;
+  }
+};
+
+class DataflowDstSafetyAnalysis
+    : public DstSafetyAnalysis,
+      public DataflowAnalysis<DataflowDstSafetyAnalysis, DstState,
+                              /*Backward=*/true, DstStatePrinter> {
+  using DFParent = DataflowAnalysis<DataflowDstSafetyAnalysis, DstState, true,
+                                    DstStatePrinter>;
+  friend DFParent;
+
+  using DstSafetyAnalysis::BC;
+  using DstSafetyAnalysis::computeNext;
+
+public:
+  DataflowDstSafetyAnalysis(BinaryFunction &BF,
+                            MCPlusBuilder::AllocatorIdTy AllocId,
+                            ArrayRef<MCPhysReg> RegsToTrackInstsFor)
+      : DstSafetyAnalysis(BF, RegsToTrackInstsFor), DFParent(BF, AllocId) {}
+
+  const DstState &getStateAfter(const MCInst &Inst) const override {
+    // The dataflow analysis base class iterates backwards over the
+    // instructions, thus "after" vs. "before" difference.
+    return DFParent::getStateBefore(Inst).get();
+  }
+
+  void run() override {
+    for (BinaryBasicBlock &BB : Func) {
+      if (auto CheckerInfo = BC.MIB->getAuthCheckedReg(BB)) {
+        LLVM_DEBUG({
+          dbgs() << "Found pointer checking sequence in " << BB.getName()
+                 << ":\n";
+          traceReg(BC, "Checked register", CheckerInfo->first);
+          traceInst(BC, "First instruction", *CheckerInfo->second);
+        });
+        RegCheckedAt[CheckerInfo->second] = CheckerInfo->first;
+      }
+    }
+    DFParent::run();
+  }
+
+protected:
+  void preflight() {}
+
+  DstState getStartingStateAtBB(const BinaryBasicBlock &BB) {
+    // In general, the initial state should be empty, not everything-is-unsafe,
+    // to give a chance for some meaningful state to be propagated to BB from
+    // an indirectly reachable "exit basic block" ending with a return or tail
+    // call instruction.
+    //
+    // A basic block without any successors, on the other hand, can be
+    // pessimistically initialized to everything-is-unsafe: this will naturally
+    // handle both return and tail call instructions and is harmless for
+    // internal indirect branch instructions (such as computed gotos).
+    if (BB.succ_empty())
+      return createUnsafeState();
+
+    return DstState();
+  }
+
+  DstState getStartingStateAtPoint(const MCInst &Point) { return DstState(); }
+
+  void doConfluence(DstState &StateOut, const DstState &StateIn) {
+    DstStatePrinter P(BC);
+    LLVM_DEBUG({
+      dbgs() << "  DataflowDstSafetyAnalysis::Confluence(\n";
+      dbgs() << "    State 1: ";
+      P.print(dbgs(), StateOut);
+      dbgs() << "\n";
+      dbgs() << "    State 2: ";
+      P.print(dbgs(), StateIn);
+      dbgs() << ")\n";
+    });
+
+    StateOut.merge(StateIn);
+
+    LLVM_DEBUG({
+      dbgs() << "    merged state: ";
+      P.print(dbgs(), StateOut);
+      dbgs() << "\n";
+    });
+  }
+
+  StringRef getAnnotationName() const { return "DataflowDstSafetyAnalysis"; }
+};
+
+class CFGUnawareDstSafetyAnalysis : public DstSafetyAnalysis,
+                                    public CFGUnawareAnalysis<DstState> {
+  using DstSafetyAnalysis::BC;
+  BinaryFunction &BF;
+
+public:
+  CFGUnawareDstSafetyAnalysis(BinaryFunction &BF,
+                              MCPlusBuilder::AllocatorIdTy AllocId,
+                              ArrayRef<MCPhysReg> RegsToTrackInstsFor)
+      : DstSafetyAnalysis(BF, RegsToTrackInstsFor),
+        CFGUnawareAnalysis(BF, AllocId, "CFGUnawareDstSafetyAnalysis"), BF(BF) {
+  }
+
+  void run() override {
+    DstState S = createUnsafeState();
+    for (auto &I : llvm::reverse(BF.instrs())) {
+      MCInst &Inst = I.second;
+
+      // If Inst can change the control flow, we cannot be sure that the next
+      // instruction (to be executed in analyzed program) is the one processed
+      // on the previous iteration, thus pessimistically reset S before
+      // starting to analyze Inst.
+      if (BC.MIB->isCall(Inst) || BC.MIB->isBranch(Inst) ||
+          BC.MIB->isReturn(Inst)) {
+        LLVM_DEBUG({ traceInst(BC, "Control flow instruction", Inst); });
+        S = createUnsafeState();
+      }
+
+      // Attach the state *after* this instruction executes.
+      setState(Inst, S);
+
+      // Compute the next state.
+      S = computeNext(Inst, S);
+    }
+  }
+
+  const DstState &getStateAfter(const MCInst &Inst) const override {
+    return getState(Inst);
+  }
+};
+
+std::shared_ptr<DstSafetyAnalysis>
+DstSafetyAnalysis::create(BinaryFunction &BF,
+                          MCPlusBuilder::AllocatorIdTy AllocId,
+                          ArrayRef<MCPhysReg> RegsToTrackInstsFor) {
+  if (BF.hasCFG())
+    return std::make_shared<DataflowDstSafetyAnalysis>(BF, AllocId,
+                                                       RegsToTrackInstsFor);
+  return std::make_shared<CFGUnawareDstSafetyAnalysis>(BF, AllocId,
+                                                       RegsToTrackInstsFor);
+}
+
 // This function could return PartialReport<T>, but currently T is always
 // MCPhysReg, even though it is an implementation detail.
 static PartialReport<MCPhysReg> make_generic_report(MCInstReference Location,
@@ -808,6 +1303,37 @@ shouldReportSigningOracle(const BinaryContext &BC, const MCInstReference &Inst,
   return make_gadget_report(SigningOracleKind, Inst, *SignedReg);
 }
 
+static std::optional<PartialReport<MCPhysReg>>
+shouldReportAuthOracle(const BinaryContext &BC, const MCInstReference &Inst,
+                       const DstState &S) {
+  static const GadgetKind AuthOracleKind("authentication oracle found");
+
+  bool IsChecked = false;
+  std::optional<MCPhysReg> AuthReg =
+      BC.MIB->getWrittenAuthenticatedReg(Inst, IsChecked);
+  if (!AuthReg || IsChecked)
+    return std::nullopt;
+
+  LLVM_DEBUG({
+    traceInst(BC, "Found auth inst", Inst);
+    traceReg(BC, "Authenticated reg", *AuthReg);
+  });
+
+  if (S.empty()) {
+    LLVM_DEBUG({ dbgs() << "    DstState is empty!\n"; });
+    return make_generic_report(
+        Inst, "Warning: no state computed for an authentication instruction "
+              "(possibly unreachable)");
+  }
+
+  LLVM_DEBUG(
+      { traceRegMask(BC, "safe output registers", S.CannotEscapeUnchecked); });
+  if (S.CannotEscapeUnchecked[*AuthReg])
+    return std::nullopt;
+
+  return make_gadget_report(AuthOracleKind, Inst, *AuthReg);
+}
+
 template <typename T> static void iterateOverInstrs(BinaryFunction &BF, T Fn) {
   if (BF.hasCFG()) {
     for (BinaryBasicBlock &BB : BF)
@@ -889,6 +1415,52 @@ void FunctionAnalysisContext::augmentUnsafeUseReports(
   }
 }
 
+void FunctionAnalysisContext::findUnsafeDefs(
+    SmallVector<PartialReport<MCPhysReg>> &Reports) {
+  if (PacRetGadgetsOnly)
+    return;
+
+  auto Analysis = DstSafetyAnalysis::create(BF, AllocatorId, {});
+  LLVM_DEBUG({ dbgs() << "Running dst register safety analysis...\n"; });
+  Analysis->run();
+  LLVM_DEBUG({
+    dbgs() << "After dst register safety analysis:\n";
+    BF.dump();
+  });
+
+  iterateOverInstrs(BF, [&](MCInstReference Inst) {
+    const DstState &S = Analysis->getStateAfter(Inst);
+
+    if (auto Report = shouldReportAuthOracle(BC, Inst, S))
+      Reports.push_back(*Report);
+  });
+}
+
+void FunctionAnalysisContext::augmentUnsafeDefReports(
+    ArrayRef<PartialReport<MCPhysReg>> Reports) {
+  SmallVector<MCPhysReg> RegsToTrack = collectRegsToTrack(Reports);
+  // Re-compute the analysis with register tracking.
+  auto Analysis = DstSafetyAnalysis::create(BF, AllocatorId, RegsToTrack);
+  LLVM_DEBUG(
+      { dbgs() << "\nRunning detailed dst register safety analysis...\n"; });
+  Analysis->run();
+  LLVM_DEBUG({
+    dbgs() << "After detailed dst register safety analysis:\n";
+    BF.dump();
+  });
+
+  // Augment gadget reports.
+  for (auto &Report : Reports) {
+    MCInstReference Location = Report.Issue->Location;
+    LLVM_DEBUG({ traceInst(BC, "Attaching leakage info to", Location); });
+    assert(Report.RequestedDetails &&
+           "Should be removed by handleSimpleReports");
+    auto DetailedInfo = std::make_shared<LeakageInfo>(
+        Analysis->getLeakingInsts(Location, BF, *Report.RequestedDetails));
+    Result.Diagnostics.emplace_back(Report.Issue, DetailedInfo);
+  }
+}
+
 void FunctionAnalysisContext::handleSimpleReports(
     SmallVector<PartialReport<MCPhysReg>> &Reports) {
   // Before re-running the detailed analysis, process the reports which do not
@@ -912,6 +1484,12 @@ void FunctionAnalysisContext::run() {
   handleSimpleReports(UnsafeUses);
   if (!UnsafeUses.empty())
     augmentUnsafeUseReports(UnsafeUses);
+
+  SmallVector<PartialReport<MCPhysReg>> UnsafeDefs;
+  findUnsafeDefs(UnsafeDefs);
+  handleSimpleReports(UnsafeDefs);
+  if (!UnsafeDefs.empty())
+    augmentUnsafeDefReports(UnsafeDefs);
 }
 
 void Analysis::runOnFunction(BinaryFunction &BF,
@@ -1015,6 +1593,12 @@ void ClobberingInfo::print(raw_ostream &OS,
   printRelatedInstrs(OS, Location, ClobberingInstrs);
 }
 
+void LeakageInfo::print(raw_ostream &OS, const MCInstReference Location) const {
+  OS << "  The " << LeakingInstrs.size()
+     << " instructions that leak the affected registers are:\n";
+  printRelatedInstrs(OS, Location, LeakingInstrs);
+}
+
 void GenericDiagnostic::generateReport(raw_ostream &OS,
                                        const BinaryContext &BC) const {
   printBasicInfo(OS, BC, Text);
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
new file mode 100644
index 000000000000..717bf40df3d0
--- /dev/null
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-authentication-oracles.s
@@ -0,0 +1,812 @@
+// RUN: %clang %cflags -march=armv8.3-a %s -o %t.exe
+// RUN: llvm-bolt-binary-analysis --scanners=pacret %t.exe 2>&1 | FileCheck -check-prefix=PACRET %s
+// RUN: llvm-bolt-binary-analysis --scanners=pauth  %t.exe 2>&1 | FileCheck %s
+
+// The detection of compiler-generated explicit pointer checks is tested in
+// gs-pauth-address-checks.s, for that reason only test here "dummy-load" and
+// "high-bits-notbi" checkers, as the shortest examples of checkers that are
+// detected per-instruction and per-BB.
+
+// PACRET-NOT: authentication oracle found in function
+
+        .text
+
+        .type   sym,@function
+sym:
+        ret
+        .size sym, .-sym
+
+        .globl  callee
+        .type   callee,@function
+callee:
+        ret
+        .size callee, .-callee
+
+        .globl  good_ret
+        .type   good_ret,@function
+good_ret:
+// CHECK-NOT: good_ret
+        autia   x0, x1
+        ret     x0
+        .size good_ret, .-good_ret
+
+        .globl  good_call
+        .type   good_call,@function
+good_call:
+// CHECK-NOT: good_call
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        autia   x0, x1
+        blr     x0
+
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size good_call, .-good_call
+
+        .globl  good_branch
+        .type   good_branch,@function
+good_branch:
+// CHECK-NOT: good_branch
+        autia   x0, x1
+        br      x0
+        .size good_branch, .-good_branch
+
+        .globl  good_load_other_reg
+        .type   good_load_other_reg,@function
+good_load_other_reg:
+// CHECK-NOT: good_load_other_reg
+        autia   x0, x1
+        ldr     x2, [x0]
+        ret
+        .size good_load_other_reg, .-good_load_other_reg
+
+        .globl  good_load_same_reg
+        .type   good_load_same_reg,@function
+good_load_same_reg:
+// CHECK-NOT: good_load_same_reg
+        autia   x0, x1
+        ldr     x0, [x0]
+        ret
+        .size good_load_same_reg, .-good_load_same_reg
+
+        .globl  good_explicit_check
+        .type   good_explicit_check,@function
+good_explicit_check:
+// CHECK-NOT: good_explicit_check
+        autia   x0, x1
+        eor     x16, x0, x0, lsl #1
+        tbz     x16, #62, 1f
+        brk     0x1234
+1:
+        ret
+        .size good_explicit_check, .-good_explicit_check
+
+        .globl  bad_unchecked
+        .type   bad_unchecked,@function
+bad_unchecked:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unchecked, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        autia   x0, x1
+        ret
+        .size bad_unchecked, .-bad_unchecked
+
+        .globl  bad_leaked_to_subroutine
+        .type   bad_leaked_to_subroutine,@function
+bad_leaked_to_subroutine:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_leaked_to_subroutine, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      bl      callee
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   paciasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   stp     x29, x30, [sp, #-0x10]!
+// CHECK-NEXT:  {{[0-9a-f]+}}:   mov     x29, sp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   bl      callee
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x2, [x0]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldp     x29, x30, [sp], #0x10
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autiasp
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ret
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        autia   x0, x1
+        bl      callee
+        ldr     x2, [x0]
+
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size bad_leaked_to_subroutine, .-bad_leaked_to_subroutine
+
+        .globl  bad_unknown_usage_read
+        .type   bad_unknown_usage_read,@function
+bad_unknown_usage_read:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_read, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     x3, x0, x1
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   mul     x3, x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x2, [x0]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ret
+        autia   x0, x1
+        // Registers are not accessible to an attacker under Pointer
+        // Authentication threat model, until spilled to memory.
+        // Thus, reporting the below MUL instruction is a false positive, since
+        // the next LDR instruction prevents any possible spilling of x3 unless
+        // the authentication succeeded. Though, rejecting anything except for
+        // a closed list of instruction types is the intended behavior of the
+        // analysis, so this false positive is by design.
+        mul     x3, x0, x1
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_read, .-bad_unknown_usage_read
+
+        .globl  bad_store_to_memory_and_wait
+        .type   bad_store_to_memory_and_wait,@function
+bad_store_to_memory_and_wait:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_store_to_memory_and_wait, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      str     x0, [x3]
+        autia   x0, x1
+        cbz     x3, 2f
+        str     x0, [x3]
+1:
+        // The thread performs a time-consuming computation while the result of
+        // authentication is accessible in memory.
+        nop
+2:
+        ldr     x2, [x0]
+        ret
+        .size bad_store_to_memory_and_wait, .-bad_store_to_memory_and_wait
+
+// FIXME: Known false negative: if no return instruction is reachable from a
+//        program point (this probably implies an infinite loop), such
+//        instruction cannot be detected as an authentication oracle.
+        .globl  bad_store_to_memory_and_hang
+        .type   bad_store_to_memory_and_hang,@function
+bad_store_to_memory_and_hang:
+// CHECK-NOT: bad_store_to_memory_and_hang
+        autia   x0, x1
+        cbz     x3, 2f
+        str     x0, [x3]
+1:
+        // The thread loops indefinitely while the result of authentication
+        // is accessible in memory.
+        b       1b
+2:
+        ldr     x2, [x0]
+        ret
+        .size bad_store_to_memory_and_hang, .-bad_store_to_memory_and_hang
+
+        .globl  bad_unknown_usage_subreg_read
+        .type   bad_unknown_usage_subreg_read,@function
+bad_unknown_usage_subreg_read:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_subreg_read, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     w3, w0, w1
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   mul     w3, w0, w1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x2, [x0]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ret
+        autia   x0, x1
+        mul     w3, w0, w1
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_subreg_read, .-bad_unknown_usage_subreg_read
+
+        .globl  bad_unknown_usage_update
+        .type   bad_unknown_usage_update,@function
+bad_unknown_usage_update:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_update, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      movk    x0, #0x2a, lsl #16
+// CHECK-NEXT:  This happens in the following basic block:
+// CHECK-NEXT:  {{[0-9a-f]+}}:   autia   x0, x1
+// CHECK-NEXT:  {{[0-9a-f]+}}:   movk    x0, #0x2a, lsl #16
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ldr     x2, [x0]
+// CHECK-NEXT:  {{[0-9a-f]+}}:   ret
+        autia   x0, x1
+        movk    x0, #42, lsl #16 // does not overwrite x0 completely
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_update, .-bad_unknown_usage_update
+
+        .globl  good_overwrite_with_constant
+        .type   good_overwrite_with_constant,@function
+good_overwrite_with_constant:
+// CHECK-NOT: good_overwrite_with_constant
+        autia   x0, x1
+        mov     x0, #42
+        ret
+        .size good_overwrite_with_constant, .-good_overwrite_with_constant
+
+// Overwriting sensitive data by instructions with unmodelled side-effects is
+// explicitly rejected, even though this particular MRS is safe.
+        .globl  bad_overwrite_with_side_effects
+        .type   bad_overwrite_with_side_effects,@function
+bad_overwrite_with_side_effects:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_overwrite_with_side_effects, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        autia   x0, x1
+        mrs     x0, CTR_EL0
+        ret
+        .size bad_overwrite_with_side_effects, .-bad_overwrite_with_side_effects
+
+// Here the new value written by MUL to x0 is completely unrelated to the result
+// of authentication, so this is a false positive.
+// FIXME: Can/should we generalize overwriting by constant to handle such cases?
+        .globl  good_unknown_overwrite
+        .type   good_unknown_overwrite,@function
+good_unknown_overwrite:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function good_unknown_overwrite, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        autia   x0, x1
+        mul     x0, x1, x2
+        ret
+        .size good_unknown_overwrite, .-good_unknown_overwrite
+
+// This is a false positive: when a general-purpose register is written to as
+// a 32-bit register, its top 32 bits are zeroed, but according to LLVM
+// representation, the instruction only overwrites the Wn register.
+        .globl  good_wreg_overwrite
+        .type   good_wreg_overwrite,@function
+good_wreg_overwrite:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function good_wreg_overwrite, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+        autia   x0, x1
+        mov     w0, #42
+        ret
+        .size good_wreg_overwrite, .-good_wreg_overwrite
+
+        .globl  good_address_arith
+        .type   good_address_arith,@function
+good_address_arith:
+// CHECK-NOT: good_address_arith
+        autia   x0, x1
+
+        add     x1, x0, #8
+        sub     x2, x1, #16
+        mov     x3, x2
+
+        ldr     x4, [x3]
+        mov     x0, #0
+        mov     x1, #0
+        mov     x2, #0
+
+        ret
+        .size good_address_arith, .-good_address_arith
+
+        .globl  good_ret_multi_bb
+        .type   good_ret_multi_bb,@function
+good_ret_multi_bb:
+// CHECK-NOT: good_ret_multi_bb
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        ret     x0
+        .size good_ret_multi_bb, .-good_ret_multi_bb
+
+        .globl  good_call_multi_bb
+        .type   good_call_multi_bb,@function
+good_call_multi_bb:
+// CHECK-NOT: good_call_multi_bb
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        blr     x0
+        cbz     x1, 2f
+        nop
+2:
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size good_call_multi_bb, .-good_call_multi_bb
+
+        .globl  good_branch_multi_bb
+        .type   good_branch_multi_bb,@function
+good_branch_multi_bb:
+// CHECK-NOT: good_branch_multi_bb
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        br      x0
+        .size good_branch_multi_bb, .-good_branch_multi_bb
+
+        .globl  good_load_other_reg_multi_bb
+        .type   good_load_other_reg_multi_bb,@function
+good_load_other_reg_multi_bb:
+// CHECK-NOT: good_load_other_reg_multi_bb
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        ldr     x2, [x0]
+        cbz     x1, 2f
+        nop
+2:
+        ret
+        .size good_load_other_reg_multi_bb, .-good_load_other_reg_multi_bb
+
+        .globl  good_load_same_reg_multi_bb
+        .type   good_load_same_reg_multi_bb,@function
+good_load_same_reg_multi_bb:
+// CHECK-NOT: good_load_same_reg_multi_bb
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        ldr     x0, [x0]
+        cbz     x1, 2f
+        nop
+2:
+        ret
+        .size good_load_same_reg_multi_bb, .-good_load_same_reg_multi_bb
+
+        .globl  good_explicit_check_multi_bb
+        .type   good_explicit_check_multi_bb,@function
+good_explicit_check_multi_bb:
+// CHECK-NOT: good_explicit_check_multi_bb
+        autia   x0, x1
+        cbz     x1, 1f
+        nop
+1:
+        eor     x16, x0, x0, lsl #1
+        tbz     x16, #62, 2f
+        brk     0x1234
+2:
+        cbz     x1, 3f
+        nop
+3:
+        ret
+        .size good_explicit_check_multi_bb, .-good_explicit_check_multi_bb
+
+        .globl  bad_unchecked_multi_bb
+        .type   bad_unchecked_multi_bb,@function
+bad_unchecked_multi_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unchecked_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        autia   x0, x1
+        cbz     x1, 1f
+        ldr     x2, [x0]
+1:
+        ret
+        .size bad_unchecked_multi_bb, .-bad_unchecked_multi_bb
+
+        .globl  bad_leaked_to_subroutine_multi_bb
+        .type   bad_leaked_to_subroutine_multi_bb,@function
+bad_leaked_to_subroutine_multi_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_leaked_to_subroutine_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      bl      callee
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        autia   x0, x1
+        cbz     x1, 1f
+        ldr     x2, [x0]
+1:
+        bl      callee
+        ldr     x2, [x0]
+
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size bad_leaked_to_subroutine_multi_bb, .-bad_leaked_to_subroutine_multi_bb
+
+        .globl  bad_unknown_usage_read_multi_bb
+        .type   bad_unknown_usage_read_multi_bb,@function
+bad_unknown_usage_read_multi_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_read_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     x3, x0, x1
+        autia   x0, x1
+        cbz     x3, 1f
+        mul     x3, x0, x1
+1:
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_read_multi_bb, .-bad_unknown_usage_read_multi_bb
+
+        .globl  bad_unknown_usage_subreg_read_multi_bb
+        .type   bad_unknown_usage_subreg_read_multi_bb,@function
+bad_unknown_usage_subreg_read_multi_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_subreg_read_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     w3, w0, w1
+        autia   x0, x1
+        cbz     x3, 1f
+        mul     w3, w0, w1
+1:
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_subreg_read_multi_bb, .-bad_unknown_usage_subreg_read_multi_bb
+
+        .globl  bad_unknown_usage_update_multi_bb
+        .type   bad_unknown_usage_update_multi_bb,@function
+bad_unknown_usage_update_multi_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_update_multi_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      movk    x0, #0x2a, lsl #16
+        autia   x0, x1
+        cbz     x3, 1f
+        movk    x0, #42, lsl #16  // does not overwrite x0 completely
+1:
+        ldr     x2, [x0]
+        ret
+        .size bad_unknown_usage_update_multi_bb, .-bad_unknown_usage_update_multi_bb
+
+        .globl  good_overwrite_with_constant_multi_bb
+        .type   good_overwrite_with_constant_multi_bb,@function
+good_overwrite_with_constant_multi_bb:
+// CHECK-NOT: good_overwrite_with_constant_multi_bb
+        autia   x0, x1
+        cbz     x3, 1f
+1:
+        mov     x0, #42
+        ret
+        .size good_overwrite_with_constant_multi_bb, .-good_overwrite_with_constant_multi_bb
+
+        .globl  good_address_arith_multi_bb
+        .type   good_address_arith_multi_bb,@function
+good_address_arith_multi_bb:
+// CHECK-NOT: good_address_arith_multi_bb
+        autia   x0, x1
+        cbz     x3, 1f
+
+        add     x1, x0, #8
+        sub     x2, x1, #16
+        mov     x0, x2
+
+        mov     x1, #0
+        mov     x2, #0
+1:
+        ldr     x3, [x0]
+        ret
+        .size good_address_arith_multi_bb, .-good_address_arith_multi_bb
+
+// FIXME: Most *_nocfg test cases contain paciasp+autiasp instructions even if
+//        LR is not spilled - this is a workaround for RET instructions being
+//        reported as non-protected, because LR state is reset at every label.
+
+        .globl  good_ret_nocfg
+        .type   good_ret_nocfg,@function
+good_ret_nocfg:
+// CHECK-NOT: good_ret_nocfg
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+
+        ret     x0
+        .size good_ret_nocfg, .-good_ret_nocfg
+
+        .globl  good_call_nocfg
+        .type   good_call_nocfg,@function
+good_call_nocfg:
+// CHECK-NOT: good_call_nocfg
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        blr     x0
+
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size good_call_nocfg, .-good_call_nocfg
+
+        .globl  good_branch_nocfg
+        .type   good_branch_nocfg,@function
+good_branch_nocfg:
+// CHECK-NOT: good_branch_nocfg
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        br      x0
+        .size good_branch_nocfg, .-good_branch_nocfg
+
+        .globl  good_load_other_reg_nocfg
+        .type   good_load_other_reg_nocfg,@function
+good_load_other_reg_nocfg:
+// CHECK-NOT: good_load_other_reg_nocfg
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        ldr     x2, [x0]
+
+        autiasp
+        ret
+        .size good_load_other_reg_nocfg, .-good_load_other_reg_nocfg
+
+        .globl  good_load_same_reg_nocfg
+        .type   good_load_same_reg_nocfg,@function
+good_load_same_reg_nocfg:
+// CHECK-NOT: good_load_same_reg_nocfg
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        ldr     x0, [x0]
+
+        autiasp
+        ret
+        .size good_load_same_reg_nocfg, .-good_load_same_reg_nocfg
+
+// FIXME: Multi-instruction checker sequences are not supported without CFG.
+
+        .globl  bad_unchecked_nocfg
+        .type   bad_unchecked_nocfg,@function
+bad_unchecked_nocfg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unchecked_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+
+        autiasp
+        ret
+        .size bad_unchecked_nocfg, .-bad_unchecked_nocfg
+
+        .globl  bad_leaked_to_subroutine_nocfg
+        .type   bad_leaked_to_subroutine_nocfg,@function
+bad_leaked_to_subroutine_nocfg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_leaked_to_subroutine_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      bl      callee # Offset: 24
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        bl      callee
+        ldr     x2, [x0]
+
+        ldp     x29, x30, [sp], #16
+        autiasp
+        ret
+        .size bad_leaked_to_subroutine_nocfg, .-bad_leaked_to_subroutine_nocfg
+
+        .globl  bad_unknown_usage_read_nocfg
+        .type   bad_unknown_usage_read_nocfg,@function
+bad_unknown_usage_read_nocfg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_read_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     x3, x0, x1
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        mul     x3, x0, x1
+        ldr     x2, [x0]
+
+        autiasp
+        ret
+        .size bad_unknown_usage_read_nocfg, .-bad_unknown_usage_read_nocfg
+
+        .globl  bad_unknown_usage_subreg_read_nocfg
+        .type   bad_unknown_usage_subreg_read_nocfg,@function
+bad_unknown_usage_subreg_read_nocfg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_subreg_read_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      mul     w3, w0, w1
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        mul     w3, w0, w1
+        ldr     x2, [x0]
+
+        autiasp
+        ret
+        .size bad_unknown_usage_subreg_read_nocfg, .-bad_unknown_usage_subreg_read_nocfg
+
+        .globl  bad_unknown_usage_update_nocfg
+        .type   bad_unknown_usage_update_nocfg,@function
+bad_unknown_usage_update_nocfg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_unknown_usage_update_nocfg, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autia   x0, x1
+// CHECK-NEXT:  The 1 instructions that leak the affected registers are:
+// CHECK-NEXT:  1.     {{[0-9a-f]+}}:      movk    x0, #0x2a, lsl #16
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        movk    x0, #42, lsl #16  // does not overwrite x0 completely
+        ldr     x2, [x0]
+
+        autiasp
+        ret
+        .size bad_unknown_usage_update_nocfg, .-bad_unknown_usage_update_nocfg
+
+        .globl  good_overwrite_with_constant_nocfg
+        .type   good_overwrite_with_constant_nocfg,@function
+good_overwrite_with_constant_nocfg:
+// CHECK-NOT: good_overwrite_with_constant_nocfg
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        mov     x0, #42
+
+        autiasp
+        ret
+        .size good_overwrite_with_constant_nocfg, .-good_overwrite_with_constant_nocfg
+
+        .globl  good_address_arith_nocfg
+        .type   good_address_arith_nocfg,@function
+good_address_arith_nocfg:
+// CHECK-NOT: good_address_arith_nocfg
+        paciasp
+        adr     x2, 1f
+        br      x2
+1:
+        autia   x0, x1
+        add     x1, x0, #8
+        sub     x2, x1, #16
+        mov     x3, x2
+
+        ldr     x4, [x3]
+        mov     x0, #0
+        mov     x1, #0
+        mov     x2, #0
+
+        autiasp
+        ret
+        .size good_address_arith_nocfg, .-good_address_arith_nocfg
+
+        .globl  good_explicit_check_unrelated_reg
+        .type   good_explicit_check_unrelated_reg,@function
+good_explicit_check_unrelated_reg:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function good_explicit_check_unrelated_reg, basic block {{[^,]+}}, at address
+        // FIXME: The below instruction is not an authentication oracle
+        autia   x2, x3    // One of possible execution paths after this instruction
+                          // ends at BRK below, thus BRK used as a trap instruction
+                          // should formally "check everything" not to introduce
+                          // false-positive here.
+        autia   x0, x1
+        eor     x16, x0, x0, lsl #1
+        tbz     x16, #62, 1f
+        brk     0x1234
+1:
+        ldr     x4, [x2]  // Right before this instruction X2 is checked - this
+                          // should be propagated to the basic block ending with
+                          // TBZ instruction above.
+        ret
+        .size good_explicit_check_unrelated_reg, .-good_explicit_check_unrelated_reg
+
+// The last BB (in layout order) is processed first by the data-flow analysis.
+// Its initial state is usually filled in a special way (because it ends with
+// `ret` instruction), and then affects the state propagated to the other BBs
+// Thus, the case of the last instruction in a function being a jump somewhere
+// in the middle is special.
+
+        .globl  good_no_ret_from_last_bb
+        .type   good_no_ret_from_last_bb,@function
+good_no_ret_from_last_bb:
+// CHECK-NOT: good_no_ret_from_last_bb
+        paciasp
+        autiasp     // authenticates LR
+        b       2f
+1:
+        ret
+2:
+        b       1b  // LR is dereferenced by `ret`, which is executed next
+        .size good_no_ret_from_last_bb, .-good_no_ret_from_last_bb
+
+        .globl  bad_no_ret_from_last_bb
+        .type   bad_no_ret_from_last_bb,@function
+bad_no_ret_from_last_bb:
+// CHECK-LABEL: GS-PAUTH: authentication oracle found in function bad_no_ret_from_last_bb, basic block {{[^,]+}}, at address
+// CHECK-NEXT:  The instruction is     {{[0-9a-f]+}}:      autiasp
+// CHECK-NEXT:  The 0 instructions that leak the affected registers are:
+        paciasp
+        autiasp     // authenticates LR
+        b       2f
+1:
+        ret     x0
+2:
+        b       1b  // X0 (but not LR) is dereferenced by `ret x0`
+        .size bad_no_ret_from_last_bb, .-bad_no_ret_from_last_bb
+
+// Test that combined auth+something instructions are not reported as
+// authentication oracles.
+
+        .globl  inst_retaa
+        .type   inst_retaa,@function
+inst_retaa:
+// CHECK-NOT: inst_retaa
+        paciasp
+        retaa
+        .size inst_retaa, .-inst_retaa
+
+        .globl  inst_blraa
+        .type   inst_blraa,@function
+inst_blraa:
+// CHECK-NOT: inst_blraa
+        paciasp
+        stp     x29, x30, [sp, #-16]!
+        mov     x29, sp
+
+        blraa   x0, x1
+
+        ldp     x29, x30, [sp], #16
+        retaa
+        .size inst_blraa, .-inst_blraa
+
+        .globl  inst_braa
+        .type   inst_braa,@function
+inst_braa:
+// CHECK-NOT: inst_braa
+        braa    x0, x1
+        .size inst_braa, .-inst_braa
+
+        .globl  inst_ldraa_no_wb
+        .type   inst_ldraa_no_wb,@function
+inst_ldraa_no_wb:
+// CHECK-NOT: inst_ldraa_no_wb
+        ldraa   x1, [x0]
+        ret
+        .size inst_ldraa_no_wb, .-inst_ldraa_no_wb
+
+        .globl  inst_ldraa_wb
+        .type   inst_ldraa_wb,@function
+inst_ldraa_wb:
+// CHECK-NOT: inst_ldraa_wb
+        ldraa   x1, [x0]!
+        ret
+        .size inst_ldraa_wb, .-inst_ldraa_wb
+
+        .globl  main
+        .type   main,@function
+main:
+        mov     x0, 0
+        ret
+        .size   main, .-main
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
index 82494d834a15..686557eb1e52 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
@@ -113,7 +113,7 @@ simple:
 // CHECK-EMPTY:
 // PAUTH-NEXT:   Found sign inst:     00000000:        paciasp # DataflowSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: >
 // PAUTH-NEXT:     Signed reg: LR
-// PAUTH-NEXT:     TrustedRegs: LR W30 W30_HI
+// PAUTH-NEXT:     TrustedRegs: LR W30 W30_HI{{[ \t]*$}}
 // PAUTH-NEXT:   Found call inst:     00000000:        blr     x0 # DataflowSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: >
 // PAUTH-NEXT:     Call destination reg: X0
 // PAUTH-NEXT:     SafeToDerefRegs: W0 X0 W0_HI{{[ \t]*$}}
@@ -220,10 +220,10 @@ nocfg:
 // CHECK-EMPTY:
 // PAUTH-NEXT:   Found call inst:     00000000:        br      x0 # UNKNOWN CONTROL FLOW # Offset: 4 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: >
 // PAUTH-NEXT:     Call destination reg: X0
-// PAUTH-NEXT:     SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI
+// PAUTH-NEXT:     SafeToDerefRegs: LR W0 W30 X0 W0_HI W30_HI{{[ \t]*$}}
 // CHECK-NEXT:   Found RET inst:     00000000:         ret # Offset: 8 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: >
 // CHECK-NEXT:     RetReg: LR
-// CHECK-NEXT:     SafeToDerefRegs:
+// CHECK-NEXT:     SafeToDerefRegs:{{[ \t]*$}}
 // CHECK-EMPTY:
 // CHECK-NEXT: Running detailed src register safety analysis...
 // CHECK-NEXT:   SrcSafetyAnalysis::ComputeNext(   adr     x0, __ENTRY_nocfg@0x[[ENTRY_ADDR]], src-state<SafeToDerefRegs: LR W30 W30_HI , TrustedRegs: LR W30 W30_HI , Insts: [0]()>)
@@ -251,6 +251,84 @@ nocfg:
 // CHECK-EMPTY:
 // CHECK-NEXT:   Attaching clobbering info to:     00000000:   ret # Offset: 8 # CFGUnawareSrcSafetyAnalysis: src-state<SafeToDerefRegs: BitVector, TrustedRegs: BitVector, Insts: [0]()>
 
+        .globl  auth_oracle
+        .type   auth_oracle,@function
+auth_oracle:
+        autia   x0, x1
+        ret
+        .size auth_oracle, .-auth_oracle
+
+// CHECK-LABEL:Analyzing function auth_oracle, AllocatorId = 1
+// CHECK-NEXT: Binary Function "auth_oracle"  {
+// CHECK-NEXT:   Number      : 4
+// CHECK-NEXT:   State       : CFG constructed
+// ...
+// CHECK:        BB Layout   : [[BB0:[0-9a-zA-Z.]+]]
+// CHECK-NEXT: }
+// CHECK-NEXT: [[BB0]] (2 instructions, align : 1)
+// CHECK-NEXT:   Entry Point
+// CHECK-NEXT:     00000000:   autia   x0, x1
+// CHECK-NEXT:     00000004:   ret
+// CHECK-EMPTY:
+// CHECK-NEXT: DWARF CFI Instructions:
+// CHECK-NEXT:     <empty>
+// CHECK-NEXT: End of Function "auth_oracle"
+// CHECK-EMPTY:
+// CHECK-NEXT: Running src register safety analysis...
+// ...
+// CHECK:      After src register safety analysis:
+// CHECK-NEXT: Binary Function "auth_oracle"  {
+// ...
+// CHECK:      End of Function "auth_oracle"
+// ...
+// PAUTH:      Running dst register safety analysis...
+// PAUTH-NEXT:   DstSafetyAnalysis::ComputeNext(       ret     x30, dst-state<CannotEscapeUnchecked: , Insts: >)
+// PAUTH-NEXT:     .. result: (dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: >)
+// PAUTH-NEXT:   DstSafetyAnalysis::ComputeNext(       autia   x0, x1, dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: >)
+// PAUTH-NEXT:     .. result: (dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: >)
+// PAUTH-NEXT: After dst register safety analysis:
+// PAUTH-NEXT: Binary Function "auth_oracle"  {
+// PAUTH-NEXT:   Number      : 4
+// PAUTH-NEXT:   State       : CFG constructed
+// ...
+// PAUTH:        BB Layout   : [[BB0]]
+// PAUTH-NEXT: }
+// PAUTH-NEXT: [[BB0]] (2 instructions, align : 1)
+// PAUTH-NEXT:   Entry Point
+// PAUTH-NEXT:     00000000:   autia   x0, x1 # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: >
+// PAUTH-NEXT:     00000004:   ret # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: >
+// PAUTH-EMPTY:
+// PAUTH-NEXT: DWARF CFI Instructions:
+// PAUTH-NEXT:     <empty>
+// PAUTH-NEXT: End of Function "auth_oracle"
+// PAUTH-EMPTY:
+// PAUTH-NEXT:   Found auth inst:     00000000:        autia   x0, x1 # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: >
+// PAUTH-NEXT:     Authenticated reg: X0
+// PAUTH-NEXT:     safe output registers: LR W30 W30_HI{{[ \t]*$}}
+// PAUTH-EMPTY:
+// PAUTH-NEXT: Running detailed dst register safety analysis...
+// PAUTH-NEXT:   DstSafetyAnalysis::ComputeNext(       ret     x30, dst-state<CannotEscapeUnchecked: , Insts: [0]()>)
+// PAUTH-NEXT:     .. result: (dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: [0]()>)
+// PAUTH-NEXT:   DstSafetyAnalysis::ComputeNext(       autia   x0, x1, dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: [0]()>)
+// PAUTH-NEXT:     .. result: (dst-state<CannotEscapeUnchecked: LR W30 W30_HI , Insts: [0](0x{{[0-9a-f]+}} )>)
+// PAUTH-NEXT: After detailed dst register safety analysis:
+// PAUTH-NEXT: Binary Function "auth_oracle"  {
+// PAUTH-NEXT:   Number      : 4
+// PAUTH-NEXT:   State       : CFG constructed
+// ...
+// PAUTH:        BB Layout   : [[BB0]]
+// PAUTH-NEXT: }
+// PAUTH-NEXT: [[BB0]] (2 instructions, align : 1)
+// PAUTH-NEXT:   Entry Point
+// PAUTH-NEXT:     00000000:   autia   x0, x1 # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: [0](0x{{[0-9a-f]+}} )>
+// PAUTH-NEXT:     00000004:   ret # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: [0]()>
+// PAUTH-EMPTY:
+// PAUTH-NEXT: DWARF CFI Instructions:
+// PAUTH-NEXT:     <empty>
+// PAUTH-NEXT: End of Function "auth_oracle"
+// PAUTH-EMPTY:
+// PAUTH-NEXT:   Attaching leakage info to:     00000000:      autia   x0, x1 # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: [0](0x{{[0-9a-f]+}} )>
+
 // CHECK-LABEL:Analyzing function main, AllocatorId = 1
         .globl  main
         .type   main,@function
-- 
cgit v1.2.3


From 936c5566db013225dc098ff961395bb19e1bf2a4 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Thu, 19 Jun 2025 07:18:21 -0500
Subject: [flang][OpenMP] Handle REQUIRES ADMO in lowering (#144362)

The previous approach rewrote the atomic constructs in the AST based on
the REQUIRES ATOMIC_DEFAULT_MEM_ORDER directives. The new approach
checks for incorrect uses of REQUIRED ADMO in the semantic analysis, and
applies it in lowering, eliminating the need for a separate
tree-rewriting procedure.
---
 flang/lib/Lower/OpenMP/OpenMP.cpp                  | 206 +++++++++++++++------
 flang/lib/Semantics/CMakeLists.txt                 |   1 -
 flang/lib/Semantics/check-omp-structure.cpp        |  19 ++
 flang/lib/Semantics/check-omp-structure.h          |   1 +
 flang/lib/Semantics/rewrite-directives.cpp         | 172 -----------------
 flang/lib/Semantics/rewrite-directives.h           |  24 ---
 flang/lib/Semantics/rewrite-parse-tree.cpp         |   4 +-
 flang/test/Lower/OpenMP/requires-admo-acqrel.f90   |  19 ++
 flang/test/Lower/OpenMP/requires-admo-invalid1.f90 |  16 ++
 flang/test/Lower/OpenMP/requires-admo-invalid2.f90 |  16 ++
 flang/test/Semantics/OpenMP/requires-atomic01.f90  | 121 ------------
 flang/test/Semantics/OpenMP/requires-atomic02.f90  | 121 ------------
 12 files changed, 220 insertions(+), 500 deletions(-)
 delete mode 100644 flang/lib/Semantics/rewrite-directives.cpp
 delete mode 100644 flang/lib/Semantics/rewrite-directives.h
 create mode 100644 flang/test/Lower/OpenMP/requires-admo-acqrel.f90
 create mode 100644 flang/test/Lower/OpenMP/requires-admo-invalid1.f90
 create mode 100644 flang/test/Lower/OpenMP/requires-admo-invalid2.f90
 delete mode 100644 flang/test/Semantics/OpenMP/requires-atomic01.f90
 delete mode 100644 flang/test/Semantics/OpenMP/requires-atomic02.f90

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 3e865a1ee718..7ad886959727 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2722,58 +2722,129 @@ static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
   return nullptr;
 }
 
-static mlir::omp::ClauseMemoryOrderKindAttr
-getAtomicMemoryOrder(lower::AbstractConverter &converter,
-                     semantics::SemanticsContext &semaCtx,
-                     const List<Clause> &clauses) {
-  std::optional<mlir::omp::ClauseMemoryOrderKind> kind;
+static mlir::omp::ClauseMemoryOrderKind
+getMemoryOrderKind(common::OmpMemoryOrderType kind) {
+  switch (kind) {
+  case common::OmpMemoryOrderType::Acq_Rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case common::OmpMemoryOrderType::Acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case common::OmpMemoryOrderType::Relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case common::OmpMemoryOrderType::Release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case common::OmpMemoryOrderType::Seq_Cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  }
+  llvm_unreachable("Unexpected kind");
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderKind(llvm::omp::Clause clauseId) {
+  switch (clauseId) {
+  case llvm::omp::Clause::OMPC_acq_rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case llvm::omp::Clause::OMPC_acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case llvm::omp::Clause::OMPC_relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case llvm::omp::Clause::OMPC_release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case llvm::omp::Clause::OMPC_seq_cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  default:
+    return std::nullopt;
+  }
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderFromRequires(const semantics::Scope &scope) {
+  // The REQUIRES construct is only allowed in the main program scope
+  // and module scope, but seems like we also accept it in a subprogram
+  // scope.
+  // For safety, traverse all enclosing scopes and check if their symbol
+  // contains REQUIRES.
+  for (const auto *sc{&scope}; sc->kind() != semantics::Scope::Kind::Global;
+       sc = &sc->parent()) {
+    const semantics::Symbol *sym = sc->symbol();
+    if (!sym)
+      continue;
+
+    const common::OmpMemoryOrderType *admo = common::visit(
+        [](auto &&s) {
+          using WithOmpDeclarative = semantics::WithOmpDeclarative;
+          if constexpr (std::is_convertible_v<decltype(s),
+                                              const WithOmpDeclarative &>) {
+            return s.ompAtomicDefaultMemOrder();
+          }
+          return static_cast<const common::OmpMemoryOrderType *>(nullptr);
+        },
+        sym->details());
+    if (admo)
+      return getMemoryOrderKind(*admo);
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getDefaultAtomicMemOrder(semantics::SemanticsContext &semaCtx) {
   unsigned version = semaCtx.langOptions().OpenMPVersion;
+  if (version > 50)
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  return std::nullopt;
+}
 
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getAtomicMemoryOrder(semantics::SemanticsContext &semaCtx,
+                     const List<Clause> &clauses,
+                     const semantics::Scope &scope) {
   for (const Clause &clause : clauses) {
-    switch (clause.id) {
-    case llvm::omp::Clause::OMPC_acq_rel:
-      kind = mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-      break;
-    case llvm::omp::Clause::OMPC_acquire:
-      kind = mlir::omp::ClauseMemoryOrderKind::Acquire;
-      break;
-    case llvm::omp::Clause::OMPC_relaxed:
-      kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
-      break;
-    case llvm::omp::Clause::OMPC_release:
-      kind = mlir::omp::ClauseMemoryOrderKind::Release;
-      break;
-    case llvm::omp::Clause::OMPC_seq_cst:
-      kind = mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-      break;
-    default:
-      break;
-    }
+    if (auto maybeKind = getMemoryOrderKind(clause.id))
+      return *maybeKind;
   }
 
-  // Starting with 5.1, if no memory-order clause is present, the effect
-  // is as if "relaxed" was present.
-  if (!kind) {
-    if (version <= 50)
-      return nullptr;
-    kind = mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  if (auto maybeKind = getMemoryOrderFromRequires(scope))
+    return *maybeKind;
+
+  return getDefaultAtomicMemOrder(semaCtx);
+}
+
+static mlir::omp::ClauseMemoryOrderKindAttr
+makeMemOrderAttr(lower::AbstractConverter &converter,
+                 std::optional<mlir::omp::ClauseMemoryOrderKind> maybeKind) {
+  if (maybeKind) {
+    return mlir::omp::ClauseMemoryOrderKindAttr::get(
+        converter.getFirOpBuilder().getContext(), *maybeKind);
   }
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  return mlir::omp::ClauseMemoryOrderKindAttr::get(builder.getContext(), *kind);
+  return nullptr;
 }
 
 static mlir::Operation * //
-genAtomicRead(lower::AbstractConverter &converter, mlir::Location loc,
+genAtomicRead(lower::AbstractConverter &converter,
+              semantics::SemanticsContext &semaCtx, mlir::Location loc,
               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
               const semantics::SomeExpr &atom,
               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-              mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+              std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
               fir::FirOpBuilder::InsertPoint preAt,
               fir::FirOpBuilder::InsertPoint atomicAt,
               fir::FirOpBuilder::InsertPoint postAt) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   builder.restoreInsertionPoint(preAt);
 
+  // If the atomic clause is read then the memory-order clause must
+  // not be release.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Release) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Acquire;
+    }
+  }
+
   mlir::Value storeAddr =
       fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
@@ -2787,7 +2858,8 @@ genAtomicRead(lower::AbstractConverter &converter, mlir::Location loc,
 
   builder.restoreInsertionPoint(atomicAt);
   mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
-      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint, memOrder);
+      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
+      makeMemOrderAttr(converter, memOrder));
 
   if (atomType != storeType) {
     lower::ExprToValueMap overrides;
@@ -2808,17 +2880,30 @@ genAtomicRead(lower::AbstractConverter &converter, mlir::Location loc,
 }
 
 static mlir::Operation * //
-genAtomicWrite(lower::AbstractConverter &converter, mlir::Location loc,
+genAtomicWrite(lower::AbstractConverter &converter,
+               semantics::SemanticsContext &semaCtx, mlir::Location loc,
                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
                const semantics::SomeExpr &atom,
                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-               mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+               std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
                fir::FirOpBuilder::InsertPoint preAt,
                fir::FirOpBuilder::InsertPoint atomicAt,
                fir::FirOpBuilder::InsertPoint postAt) {
   fir::FirOpBuilder &builder = converter.getFirOpBuilder();
   builder.restoreInsertionPoint(preAt);
 
+  // If the atomic clause is write then the memory-order clause must
+  // not be acquire.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acquire) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Release;
+    }
+  }
+
   mlir::Value value =
       fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
@@ -2826,16 +2911,17 @@ genAtomicWrite(lower::AbstractConverter &converter, mlir::Location loc,
 
   builder.restoreInsertionPoint(atomicAt);
   mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
-      loc, atomAddr, converted, hint, memOrder);
+      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
   return op;
 }
 
 static mlir::Operation *
-genAtomicUpdate(lower::AbstractConverter &converter, mlir::Location loc,
+genAtomicUpdate(lower::AbstractConverter &converter,
+                semantics::SemanticsContext &semaCtx, mlir::Location loc,
                 lower::StatementContext &stmtCtx, mlir::Value atomAddr,
                 const semantics::SomeExpr &atom,
                 const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+                std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
                 fir::FirOpBuilder::InsertPoint preAt,
                 fir::FirOpBuilder::InsertPoint atomicAt,
                 fir::FirOpBuilder::InsertPoint postAt) {
@@ -2858,8 +2944,8 @@ genAtomicUpdate(lower::AbstractConverter &converter, mlir::Location loc,
   }
 
   builder.restoreInsertionPoint(atomicAt);
-  auto updateOp =
-      builder.create<mlir::omp::AtomicUpdateOp>(loc, atomAddr, hint, memOrder);
+  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
+      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
 
   mlir::Region &region = updateOp->getRegion(0);
   mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
@@ -2878,11 +2964,12 @@ genAtomicUpdate(lower::AbstractConverter &converter, mlir::Location loc,
 }
 
 static mlir::Operation *
-genAtomicOperation(lower::AbstractConverter &converter, mlir::Location loc,
+genAtomicOperation(lower::AbstractConverter &converter,
+                   semantics::SemanticsContext &semaCtx, mlir::Location loc,
                    lower::StatementContext &stmtCtx, int action,
                    mlir::Value atomAddr, const semantics::SomeExpr &atom,
                    const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                   mlir::omp::ClauseMemoryOrderKindAttr memOrder,
+                   std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
                    fir::FirOpBuilder::InsertPoint preAt,
                    fir::FirOpBuilder::InsertPoint atomicAt,
                    fir::FirOpBuilder::InsertPoint postAt) {
@@ -2894,14 +2981,14 @@ genAtomicOperation(lower::AbstractConverter &converter, mlir::Location loc,
   // builder's insertion point, or set it to anything specific.
   switch (action) {
   case parser::OpenMPAtomicConstruct::Analysis::Read:
-    return genAtomicRead(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
-                         memOrder, preAt, atomicAt, postAt);
+    return genAtomicRead(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                         assign, hint, memOrder, preAt, atomicAt, postAt);
   case parser::OpenMPAtomicConstruct::Analysis::Write:
-    return genAtomicWrite(converter, loc, stmtCtx, atomAddr, atom, assign, hint,
-                          memOrder, preAt, atomicAt, postAt);
+    return genAtomicWrite(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                          assign, hint, memOrder, preAt, atomicAt, postAt);
   case parser::OpenMPAtomicConstruct::Analysis::Update:
-    return genAtomicUpdate(converter, loc, stmtCtx, atomAddr, atom, assign,
-                           hint, memOrder, preAt, atomicAt, postAt);
+    return genAtomicUpdate(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                           assign, hint, memOrder, preAt, atomicAt, postAt);
   default:
     return nullptr;
   }
@@ -3899,8 +3986,9 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
   mlir::Value atomAddr =
       fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
   mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
-  mlir::omp::ClauseMemoryOrderKindAttr memOrder =
-      getAtomicMemoryOrder(converter, semaCtx, clauses);
+  std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder =
+      getAtomicMemoryOrder(semaCtx, clauses,
+                           semaCtx.FindScope(construct.source));
 
   if (auto *cond = get(analysis.cond)) {
     (void)cond;
@@ -3918,8 +4006,8 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
              "Expexcing two actions");
       (void)action0;
       (void)action1;
-      captureOp =
-          builder.create<mlir::omp::AtomicCaptureOp>(loc, hint, memOrder);
+      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
+          loc, hint, makeMemOrderAttr(converter, memOrder));
       // Set the non-atomic insertion point to before the atomic.capture.
       preAt = getInsertionPointBefore(captureOp);
 
@@ -3931,7 +4019,7 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
       atomicAt = getInsertionPointBefore(term);
       postAt = getInsertionPointAfter(captureOp);
       hint = nullptr;
-      memOrder = nullptr;
+      memOrder = std::nullopt;
     } else {
       // Non-capturing operation.
       assert(action0 != analysis.None && action1 == analysis.None &&
@@ -3943,16 +4031,16 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
     // The builder's insertion point needs to be specifically set before
     // each call to `genAtomicOperation`.
     mlir::Operation *firstOp = genAtomicOperation(
-        converter, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
+        converter, semaCtx, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
         *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
     assert(firstOp && "Should have created an atomic operation");
     atomicAt = getInsertionPointAfter(firstOp);
 
     mlir::Operation *secondOp = nullptr;
     if (analysis.op1.what != analysis.None) {
-      secondOp = genAtomicOperation(converter, loc, stmtCtx, analysis.op1.what,
-                                    atomAddr, atom, *get(analysis.op1.assign),
-                                    hint, memOrder, preAt, atomicAt, postAt);
+      secondOp = genAtomicOperation(
+          converter, semaCtx, loc, stmtCtx, analysis.op1.what, atomAddr, atom,
+          *get(analysis.op1.assign), hint, memOrder, preAt, atomicAt, postAt);
     }
 
     if (construct.IsCapture()) {
diff --git a/flang/lib/Semantics/CMakeLists.txt b/flang/lib/Semantics/CMakeLists.txt
index 18c89587843a..c0fda3631c01 100644
--- a/flang/lib/Semantics/CMakeLists.txt
+++ b/flang/lib/Semantics/CMakeLists.txt
@@ -40,7 +40,6 @@ add_flang_library(FortranSemantics
   resolve-directives.cpp
   resolve-names-utils.cpp
   resolve-names.cpp
-  rewrite-directives.cpp
   rewrite-parse-tree.cpp
   runtime-type-info.cpp
   scope.cpp
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 36d4bcb5d99f..68cea6739830 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -1719,6 +1719,22 @@ void OmpStructureChecker::Leave(const parser::OpenMPDepobjConstruct &x) {
 void OmpStructureChecker::Enter(const parser::OpenMPRequiresConstruct &x) {
   const auto &dir{std::get<parser::Verbatim>(x.t)};
   PushContextAndClauseSets(dir.source, llvm::omp::Directive::OMPD_requires);
+
+  if (visitedAtomicSource_.empty()) {
+    return;
+  }
+  const auto &clauseList{std::get<parser::OmpClauseList>(x.t)};
+  for (const parser::OmpClause &clause : clauseList.v) {
+    llvm::omp::Clause id{clause.Id()};
+    if (id == llvm::omp::Clause::OMPC_atomic_default_mem_order) {
+      parser::MessageFormattedText txt(
+          "REQUIRES directive with '%s' clause found lexically after atomic operation without a memory order clause"_err_en_US,
+          parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(id)));
+      parser::Message message(clause.source, txt);
+      message.Attach(visitedAtomicSource_, "Previous atomic construct"_en_US);
+      context_.Say(std::move(message));
+    }
+  }
 }
 
 void OmpStructureChecker::Leave(const parser::OpenMPRequiresConstruct &) {
@@ -4056,6 +4072,9 @@ void OmpStructureChecker::CheckAtomicUpdate(
 }
 
 void OmpStructureChecker::Enter(const parser::OpenMPAtomicConstruct &x) {
+  if (visitedAtomicSource_.empty())
+    visitedAtomicSource_ = x.source;
+
   // All of the following groups have the "exclusive" property, i.e. at
   // most one clause from each group is allowed.
   // The exclusivity-checking code should eventually be unified for all
diff --git a/flang/lib/Semantics/check-omp-structure.h b/flang/lib/Semantics/check-omp-structure.h
index 2074ec611dc2..beb6e0528e81 100644
--- a/flang/lib/Semantics/check-omp-structure.h
+++ b/flang/lib/Semantics/check-omp-structure.h
@@ -360,6 +360,7 @@ private:
   };
   int directiveNest_[LastType + 1] = {0};
 
+  parser::CharBlock visitedAtomicSource_;
   SymbolSourceMap deferredNonVariables_;
 
   using LoopConstruct = std::variant<const parser::DoConstruct *,
diff --git a/flang/lib/Semantics/rewrite-directives.cpp b/flang/lib/Semantics/rewrite-directives.cpp
deleted file mode 100644
index 91b60ea151de..000000000000
--- a/flang/lib/Semantics/rewrite-directives.cpp
+++ /dev/null
@@ -1,172 +0,0 @@
-//===-- lib/Semantics/rewrite-directives.cpp ------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "rewrite-directives.h"
-#include "flang/Parser/parse-tree-visitor.h"
-#include "flang/Parser/parse-tree.h"
-#include "flang/Semantics/semantics.h"
-#include "flang/Semantics/symbol.h"
-#include "llvm/Frontend/OpenMP/OMP.h"
-#include <list>
-
-namespace Fortran::semantics {
-
-using namespace parser::literals;
-
-class DirectiveRewriteMutator {
-public:
-  explicit DirectiveRewriteMutator(SemanticsContext &context)
-      : context_{context} {}
-
-  // Default action for a parse tree node is to visit children.
-  template <typename T> bool Pre(T &) { return true; }
-  template <typename T> void Post(T &) {}
-
-protected:
-  SemanticsContext &context_;
-};
-
-// Rewrite atomic constructs to add an explicit memory ordering to all that do
-// not specify it, honoring in this way the `atomic_default_mem_order` clause of
-// the REQUIRES directive.
-class OmpRewriteMutator : public DirectiveRewriteMutator {
-public:
-  explicit OmpRewriteMutator(SemanticsContext &context)
-      : DirectiveRewriteMutator(context) {}
-
-  template <typename T> bool Pre(T &) { return true; }
-  template <typename T> void Post(T &) {}
-
-  bool Pre(parser::OpenMPAtomicConstruct &);
-  bool Pre(parser::OpenMPRequiresConstruct &);
-
-private:
-  bool atomicDirectiveDefaultOrderFound_{false};
-};
-
-bool OmpRewriteMutator::Pre(parser::OpenMPAtomicConstruct &x) {
-  // Find top-level parent of the operation.
-  Symbol *topLevelParent{[&]() {
-    Symbol *symbol{nullptr};
-    Scope *scope{&context_.FindScope(
-        std::get<parser::OmpDirectiveSpecification>(x.t).source)};
-    do {
-      if (Symbol * parent{scope->symbol()}) {
-        symbol = parent;
-      }
-      scope = &scope->parent();
-    } while (!scope->IsGlobal());
-
-    assert(symbol &&
-        "Atomic construct must be within a scope associated with a symbol");
-    return symbol;
-  }()};
-
-  // Get the `atomic_default_mem_order` clause from the top-level parent.
-  std::optional<common::OmpMemoryOrderType> defaultMemOrder;
-  common::visit(
-      [&](auto &details) {
-        if constexpr (std::is_convertible_v<decltype(&details),
-                          WithOmpDeclarative *>) {
-          if (details.has_ompAtomicDefaultMemOrder()) {
-            defaultMemOrder = *details.ompAtomicDefaultMemOrder();
-          }
-        }
-      },
-      topLevelParent->details());
-
-  if (!defaultMemOrder) {
-    return false;
-  }
-
-  auto findMemOrderClause{[](const parser::OmpClauseList &clauses) {
-    return llvm::any_of(
-        clauses.v, [](auto &clause) -> const parser::OmpClause * {
-          switch (clause.Id()) {
-          case llvm::omp::Clause::OMPC_acq_rel:
-          case llvm::omp::Clause::OMPC_acquire:
-          case llvm::omp::Clause::OMPC_relaxed:
-          case llvm::omp::Clause::OMPC_release:
-          case llvm::omp::Clause::OMPC_seq_cst:
-            return &clause;
-          default:
-            return nullptr;
-          }
-        });
-  }};
-
-  auto &dirSpec{std::get<parser::OmpDirectiveSpecification>(x.t)};
-  auto &clauseList{std::get<std::optional<parser::OmpClauseList>>(dirSpec.t)};
-  if (clauseList) {
-    if (findMemOrderClause(*clauseList)) {
-      return false;
-    }
-  } else {
-    clauseList = parser::OmpClauseList(decltype(parser::OmpClauseList::v){});
-  }
-
-  // Add a memory order clause to the atomic directive.
-  atomicDirectiveDefaultOrderFound_ = true;
-  llvm::omp::Clause kind{x.GetKind()};
-  switch (*defaultMemOrder) {
-  case common::OmpMemoryOrderType::Acq_Rel:
-    // FIXME: Implement 5.0 rules, pending clarification on later spec
-    // versions.
-    // [5.0:62:22-26]
-    if (kind == llvm::omp::Clause::OMPC_read) {
-      clauseList->v.emplace_back(
-          parser::OmpClause{parser::OmpClause::Acquire{}});
-    } else if (kind == llvm::omp::Clause::OMPC_update && x.IsCapture()) {
-      clauseList->v.emplace_back(
-          parser::OmpClause{parser::OmpClause::AcqRel{}});
-    } else {
-      clauseList->v.emplace_back(
-          parser::OmpClause{parser::OmpClause::Release{}});
-    }
-    break;
-  case common::OmpMemoryOrderType::Relaxed:
-    clauseList->v.emplace_back(parser::OmpClause{parser::OmpClause::Relaxed{}});
-    break;
-  case common::OmpMemoryOrderType::Seq_Cst:
-    clauseList->v.emplace_back(parser::OmpClause{parser::OmpClause::SeqCst{}});
-    break;
-  default:
-    // FIXME: Don't process other values at the moment since their validity
-    // depends on the OpenMP version (which is unavailable here).
-    break;
-  }
-
-  return false;
-}
-
-bool OmpRewriteMutator::Pre(parser::OpenMPRequiresConstruct &x) {
-  for (parser::OmpClause &clause : std::get<parser::OmpClauseList>(x.t).v) {
-    if (std::holds_alternative<parser::OmpClause::AtomicDefaultMemOrder>(
-            clause.u) &&
-        atomicDirectiveDefaultOrderFound_) {
-      context_.Say(clause.source,
-          "REQUIRES directive with '%s' clause found lexically after atomic "
-          "operation without a memory order clause"_err_en_US,
-          parser::ToUpperCaseLetters(llvm::omp::getOpenMPClauseName(
-              llvm::omp::OMPC_atomic_default_mem_order)
-                                         .str()));
-    }
-  }
-  return false;
-}
-
-bool RewriteOmpParts(SemanticsContext &context, parser::Program &program) {
-  if (!context.IsEnabled(common::LanguageFeature::OpenMP)) {
-    return true;
-  }
-  OmpRewriteMutator ompMutator{context};
-  parser::Walk(program, ompMutator);
-  return !context.AnyFatalError();
-}
-
-} // namespace Fortran::semantics
diff --git a/flang/lib/Semantics/rewrite-directives.h b/flang/lib/Semantics/rewrite-directives.h
deleted file mode 100644
index 675962192842..000000000000
--- a/flang/lib/Semantics/rewrite-directives.h
+++ /dev/null
@@ -1,24 +0,0 @@
-//===-- lib/Semantics/rewrite-directives.h ----------------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
-#define FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
-
-namespace Fortran::parser {
-struct Program;
-} // namespace Fortran::parser
-
-namespace Fortran::semantics {
-class SemanticsContext;
-} // namespace Fortran::semantics
-
-namespace Fortran::semantics {
-bool RewriteOmpParts(SemanticsContext &, parser::Program &);
-} // namespace Fortran::semantics
-
-#endif // FORTRAN_SEMANTICS_REWRITE_DIRECTIVES_H_
diff --git a/flang/lib/Semantics/rewrite-parse-tree.cpp b/flang/lib/Semantics/rewrite-parse-tree.cpp
index 577558e7e33b..4eeb1b9ed3c1 100644
--- a/flang/lib/Semantics/rewrite-parse-tree.cpp
+++ b/flang/lib/Semantics/rewrite-parse-tree.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "rewrite-parse-tree.h"
-#include "rewrite-directives.h"
+
 #include "flang/Common/indirection.h"
 #include "flang/Parser/parse-tree-visitor.h"
 #include "flang/Parser/parse-tree.h"
@@ -229,7 +229,7 @@ void RewriteMutator::Post(parser::WriteStmt &x) {
 bool RewriteParseTree(SemanticsContext &context, parser::Program &program) {
   RewriteMutator mutator{context};
   parser::Walk(program, mutator);
-  return !context.AnyFatalError() && RewriteOmpParts(context, program);
+  return !context.AnyFatalError();
 }
 
 } // namespace Fortran::semantics
diff --git a/flang/test/Lower/OpenMP/requires-admo-acqrel.f90 b/flang/test/Lower/OpenMP/requires-admo-acqrel.f90
new file mode 100644
index 000000000000..525a846f410d
--- /dev/null
+++ b/flang/test/Lower/OpenMP/requires-admo-acqrel.f90
@@ -0,0 +1,19 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+module m
+!$omp requires atomic_default_mem_order(acq_rel)
+
+contains
+
+subroutine f00(x, v)
+  integer :: x, v
+!CHECK: omp.atomic.read %{{[ %#=0-9]+}} memory_order(acquire)
+  !$omp atomic read
+    v = x
+
+!CHECK: omp.atomic.write %{{[ %#=0-9]+}} memory_order(release)
+  !$omp atomic write
+    x = v
+end
+
+end module
diff --git a/flang/test/Lower/OpenMP/requires-admo-invalid1.f90 b/flang/test/Lower/OpenMP/requires-admo-invalid1.f90
new file mode 100644
index 000000000000..b21d3bbbc786
--- /dev/null
+++ b/flang/test/Lower/OpenMP/requires-admo-invalid1.f90
@@ -0,0 +1,16 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+module m
+!$omp requires atomic_default_mem_order(acquire)
+
+contains
+
+subroutine f00(x, v)
+  integer :: x, v
+!CHECK: omp.atomic.write %{{[ %#=0-9]+}} memory_order(relaxed)
+  !$omp atomic write
+    x = v
+end
+
+end module
+
diff --git a/flang/test/Lower/OpenMP/requires-admo-invalid2.f90 b/flang/test/Lower/OpenMP/requires-admo-invalid2.f90
new file mode 100644
index 000000000000..33caa25dcc64
--- /dev/null
+++ b/flang/test/Lower/OpenMP/requires-admo-invalid2.f90
@@ -0,0 +1,16 @@
+!RUN: %flang_fc1 -emit-hlfir -fopenmp -fopenmp-version=52 %s -o - | FileCheck %s
+
+module m
+!$omp requires atomic_default_mem_order(release)
+
+contains
+
+subroutine f00(x, v)
+  integer :: x, v
+!CHECK: omp.atomic.read {{[ %#=0-9]+}} memory_order(relaxed)
+  !$omp atomic read
+    v = x
+end
+
+end module
+
diff --git a/flang/test/Semantics/OpenMP/requires-atomic01.f90 b/flang/test/Semantics/OpenMP/requires-atomic01.f90
deleted file mode 100644
index e8817c3f5ef6..000000000000
--- a/flang/test/Semantics/OpenMP/requires-atomic01.f90
+++ /dev/null
@@ -1,121 +0,0 @@
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=50 -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
-! Ensure that requires atomic_default_mem_order is used to update atomic
-! operations with no explicit memory order set.
-program requires
-  implicit none
-  !$omp requires atomic_default_mem_order(seq_cst)
-  integer :: i, j
-
-  ! ----------------------------------------------------------------------------
-  ! READ
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Read
-  ! CHECK: OmpClause -> SeqCst
-  !$omp atomic read
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Read
-  !$omp atomic relaxed read
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Read
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic read relaxed
-  i = j
-  
-  ! ----------------------------------------------------------------------------
-  ! WRITE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Write
-  ! CHECK: OmpClause -> SeqCst
-  !$omp atomic write
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Write
-  !$omp atomic relaxed write
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Write
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic write relaxed
-  i = j
-
-  ! ----------------------------------------------------------------------------
-  ! UPDATE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Update
-  ! CHECK: OmpClause -> SeqCst
-  !$omp atomic update
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Update
-  !$omp atomic relaxed update
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Update
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic update relaxed
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> SeqCst
-  !$omp atomic
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic relaxed
-  i = i + j
-
-  ! ----------------------------------------------------------------------------
-  ! CAPTURE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Capture
-  ! CHECK: OmpClause -> SeqCst
-  !$omp atomic capture
-  i = j
-  j = j + 1
-  !$omp end atomic
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Capture
-  !$omp atomic relaxed capture
-  i = j
-  j = j + 1
-  !$omp end atomic
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Capture
-  ! CHECK-NOT: OmpClause -> SeqCst
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic capture relaxed
-  i = j
-  j = j + 1
-  !$omp end atomic
-end program requires
diff --git a/flang/test/Semantics/OpenMP/requires-atomic02.f90 b/flang/test/Semantics/OpenMP/requires-atomic02.f90
deleted file mode 100644
index 04a9b7a09aa9..000000000000
--- a/flang/test/Semantics/OpenMP/requires-atomic02.f90
+++ /dev/null
@@ -1,121 +0,0 @@
-! RUN: %flang_fc1 -fopenmp -fopenmp-version=50 -fdebug-dump-parse-tree %s 2>&1 | FileCheck %s
-! Ensure that requires atomic_default_mem_order is used to update atomic
-! operations with no explicit memory order set. ACQ_REL clause tested here.
-program requires
-  implicit none
-  !$omp requires atomic_default_mem_order(acq_rel)
-  integer :: i, j
-
-  ! ----------------------------------------------------------------------------
-  ! READ
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Read
-  ! CHECK: OmpClause -> Acquire
-  !$omp atomic read
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Read
-  !$omp atomic relaxed read
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Read
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic read relaxed
-  i = j
-  
-  ! ----------------------------------------------------------------------------
-  ! WRITE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Write
-  ! CHECK: OmpClause -> Release
-  !$omp atomic write
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Write
-  !$omp atomic relaxed write
-  i = j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Write
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic write relaxed
-  i = j
-
-  ! ----------------------------------------------------------------------------
-  ! UPDATE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Update
-  ! CHECK: OmpClause -> Release
-  !$omp atomic update
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Update
-  !$omp atomic relaxed update
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Update
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic update relaxed
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Release
-  !$omp atomic
-  i = i + j
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic relaxed
-  i = i + j
-
-  ! ----------------------------------------------------------------------------
-  ! CAPTURE
-  ! ----------------------------------------------------------------------------
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK: OmpClause -> Capture
-  ! CHECK: OmpClause -> AcqRel
-  !$omp atomic capture
-  i = j
-  j = j + 1
-  !$omp end atomic
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Relaxed
-  ! CHECK: OmpClause -> Capture
-  !$omp atomic relaxed capture
-  i = j
-  j = j + 1
-  !$omp end atomic
-
-  ! CHECK-LABEL: OpenMPAtomicConstruct
-  ! CHECK-NOT: OmpClause -> AcqRel
-  ! CHECK: OmpClause -> Capture
-  ! CHECK: OmpClause -> Relaxed
-  !$omp atomic capture relaxed
-  i = j
-  j = j + 1
-  !$omp end atomic
-end program requires
-- 
cgit v1.2.3


From e478a22d540d336632fb3c110c5377447cd7f3b2 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Thu, 19 Jun 2025 13:24:39 +0100
Subject: [LLVM][IRBuilder] Use NUW arithmetic for
 Create{ElementCount,TypeSize}. (#143532)

This put the onus on the caller to ensure the result type is big enough.
In the unlikely event a cropped result is required then explicitly
truncate a safe value.
---
 clang/test/CodeGen/builtin_vectorelements.c        |   8 +-
 llvm/include/llvm/IR/IRBuilder.h                   |   6 +-
 llvm/lib/IR/IRBuilder.cpp                          |   2 +-
 llvm/test/Analysis/ValueTracking/phi-known-bits.ll |   4 +-
 .../AddressSanitizer/asan-masked-load-store.ll     |  12 +-
 .../AddressSanitizer/asan-vp-load-store.ll         |  14 +-
 .../AddressSanitizer/vector-load-store.ll          |  44 +--
 llvm/test/Instrumentation/BoundsChecking/simple.ll |  14 +-
 .../HWAddressSanitizer/vector-load-store.ll        |  20 +-
 .../MemorySanitizer/vector-load-store.ll           |  10 +-
 .../test/Instrumentation/MemorySanitizer/vscale.ll |  20 +-
 .../AArch64/sve-intrinsic-opts-counting-elems.ll   |   8 +-
 llvm/test/Transforms/InstCombine/gep-vector.ll     |   2 +-
 .../test/Transforms/InstCombine/gepofconstgepi8.ll |   2 +-
 llvm/test/Transforms/InstCombine/getelementptr.ll  |   4 +-
 llvm/test/Transforms/InstCombine/icmp-gep.ll       |   8 +-
 .../Transforms/InstCombine/masked_intrinsics.ll    |   2 +-
 llvm/test/Transforms/InstCombine/opaque-ptr.ll     |   4 +-
 .../InstCombine/scalable-vector-array.ll           |   4 +-
 .../InstCombine/scalable-vector-struct.ll          |  16 +-
 llvm/test/Transforms/InstCombine/sub-gep.ll        |   6 +-
 llvm/test/Transforms/InstCombine/vscale_gep.ll     |   6 +-
 .../LoopVectorize/AArch64/clamped-trip-count.ll    |   8 +-
 .../AArch64/conditional-branches-cost.ll           |   6 +-
 .../AArch64/divs-with-scalable-vfs.ll              |  20 +-
 .../AArch64/eliminate-tail-predication.ll          |   6 +-
 .../AArch64/gather-do-not-vectorize-addressing.ll  |   6 +-
 .../LoopVectorize/AArch64/induction-costs-sve.ll   |  20 +-
 .../AArch64/low_trip_count_predicates.ll           |  32 +--
 .../LoopVectorize/AArch64/masked-call.ll           | 100 +++----
 .../LoopVectorize/AArch64/optsize_minsize.ll       |  18 +-
 .../AArch64/outer_loop_prefer_scalable.ll          |   6 +-
 .../AArch64/partial-reduce-chained.ll              |  72 ++---
 .../AArch64/partial-reduce-dot-product-epilogue.ll |   8 +-
 .../AArch64/partial-reduce-dot-product-mixed.ll    |  40 +--
 .../AArch64/partial-reduce-dot-product.ll          | 304 ++++++++++-----------
 .../LoopVectorize/AArch64/partial-reduce-sub.ll    |  22 +-
 .../AArch64/pr60831-sve-inv-store-crash.ll         |   8 +-
 .../AArch64/reduction-recurrence-costs-sve.ll      |  44 +--
 .../AArch64/scalable-avoid-scalarization.ll        |   8 +-
 .../AArch64/scalable-reduction-inloop-cond.ll      |  12 +-
 .../LoopVectorize/AArch64/scalable-strict-fadd.ll  | 258 ++++++++---------
 .../LoopVectorize/AArch64/simple_early_exit.ll     |  12 +-
 .../LoopVectorize/AArch64/store-costs-sve.ll       |  26 +-
 .../AArch64/sve-epilog-vect-inloop-reductions.ll   |   8 +-
 .../sve-epilog-vect-no-remaining-iterations.ll     |   8 +-
 .../AArch64/sve-epilog-vect-reductions.ll          |   8 +-
 .../AArch64/sve-epilog-vect-strict-reductions.ll   |   8 +-
 .../LoopVectorize/AArch64/sve-epilog-vect.ll       |  64 ++---
 .../AArch64/sve-extract-last-veclane.ll            |   4 +-
 .../Transforms/LoopVectorize/AArch64/sve-fneg.ll   |  12 +-
 .../AArch64/sve-inductions-unusual-types.ll        |  12 +-
 .../LoopVectorize/AArch64/sve-inductions.ll        |   6 +-
 .../AArch64/sve-interleaved-accesses.ll            |  14 +-
 .../AArch64/sve-interleaved-masked-accesses.ll     |  60 ++--
 .../LoopVectorize/AArch64/sve-inv-store.ll         |  14 +-
 .../AArch64/sve-live-out-pointer-induction.ll      |   8 +-
 .../LoopVectorize/AArch64/sve-low-trip-count.ll    |   4 +-
 .../LoopVectorize/AArch64/sve-multiexit.ll         |  24 +-
 .../sve-runtime-check-size-based-threshold.ll      |  16 +-
 .../AArch64/sve-tail-folding-forced.ll             |   6 +-
 .../AArch64/sve-tail-folding-optsize.ll            |   4 +-
 .../AArch64/sve-tail-folding-reductions.ll         |  36 +--
 .../AArch64/sve-tail-folding-unroll.ll             |  54 ++--
 .../LoopVectorize/AArch64/sve-tail-folding.ll      |  60 ++--
 .../LoopVectorize/AArch64/sve-vector-reverse.ll    |  12 +-
 .../AArch64/sve-vscale-based-trip-counts.ll        |  60 ++--
 .../LoopVectorize/AArch64/sve-widen-gep.ll         |  16 +-
 .../LoopVectorize/AArch64/sve2-histcnt-epilogue.ll |  14 +-
 .../AArch64/sve2-histcnt-too-many-deps.ll          |   6 +-
 .../LoopVectorize/AArch64/tail-folding-styles.ll   |  30 +-
 .../AArch64/uniform-args-call-variants.ll          |  24 +-
 .../LoopVectorize/AArch64/wider-VF-for-callinst.ll |   4 +-
 llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll   |  12 +-
 .../RISCV/blend-any-of-reduction-cost.ll           |   6 +-
 .../RISCV/blocks-with-dead-instructions.ll         |  54 ++--
 .../LoopVectorize/RISCV/dead-ops-cost.ll           |  24 +-
 .../Transforms/LoopVectorize/RISCV/defaults.ll     |  12 +-
 llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll |  54 ++--
 llvm/test/Transforms/LoopVectorize/RISCV/f16.ll    |   6 +-
 .../Transforms/LoopVectorize/RISCV/fminimumnum.ll  |  96 +++----
 .../LoopVectorize/RISCV/induction-costs.ll         |   6 +-
 .../LoopVectorize/RISCV/inloop-reduction.ll        |  44 +--
 .../LoopVectorize/RISCV/interleaved-accesses.ll    |  84 +++---
 .../RISCV/interleaved-masked-access.ll             |  22 +-
 llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll   |  18 +-
 .../LoopVectorize/RISCV/low-trip-count.ll          |   4 +-
 .../LoopVectorize/RISCV/mask-index-type.ll         |   6 +-
 .../LoopVectorize/RISCV/masked_gather_scatter.ll   |  12 +-
 .../RISCV/partial-reduce-dot-product.ll            |  48 ++--
 ...pinstruction-or-drop-poison-generating-flags.ll |   6 +-
 .../LoopVectorize/RISCV/remark-reductions.ll       |   6 +-
 .../RISCV/riscv-vector-reverse-output.ll           |  36 +--
 .../LoopVectorize/RISCV/riscv-vector-reverse.ll    |  28 +-
 .../LoopVectorize/RISCV/safe-dep-distance.ll       |  18 +-
 .../LoopVectorize/RISCV/scalable-basics.ll         |  72 ++---
 .../LoopVectorize/RISCV/scalable-tailfold.ll       |  24 +-
 .../LoopVectorize/RISCV/select-cmp-reduction.ll    |  24 +-
 .../LoopVectorize/RISCV/strided-accesses.ll        |  62 ++---
 .../RISCV/truncate-to-minimal-bitwidth-cost.ll     |  22 +-
 .../truncate-to-minimal-bitwidth-evl-crash.ll      |   4 +-
 .../RISCV/type-info-cache-evl-crash.ll             |   6 +-
 .../LoopVectorize/RISCV/uniform-load-store.ll      |  78 +++---
 ...orize-force-tail-with-evl-bin-unary-ops-args.ll | 108 ++++----
 ...ectorize-force-tail-with-evl-call-intrinsics.ll |  72 ++---
 ...ectorize-force-tail-with-evl-cast-intrinsics.ll |  70 ++---
 ...vectorize-force-tail-with-evl-cond-reduction.ll |  72 ++---
 .../RISCV/vectorize-force-tail-with-evl-div.ll     |  16 +-
 ...e-force-tail-with-evl-fixed-order-recurrence.ll |  84 +++---
 ...ctorize-force-tail-with-evl-inloop-reduction.ll | 168 ++++++------
 .../vectorize-force-tail-with-evl-interleave.ll    |  16 +-
 ...orize-force-tail-with-evl-intermediate-store.ll |  24 +-
 .../RISCV/vectorize-force-tail-with-evl-iv32.ll    |  12 +-
 ...torize-force-tail-with-evl-known-no-overflow.ll |  16 +-
 ...ctorize-force-tail-with-evl-masked-loadstore.ll |   6 +-
 ...torize-force-tail-with-evl-ordered-reduction.ll |   6 +-
 .../vectorize-force-tail-with-evl-reduction.ll     | 168 ++++++------
 ...orize-force-tail-with-evl-reverse-load-store.ll |  12 +-
 ...torize-force-tail-with-evl-safe-dep-distance.ll |   8 +-
 .../vectorize-force-tail-with-evl-uniform-store.ll |   6 +-
 .../LoopVectorize/RISCV/vectorize-vp-intrinsics.ll |  12 +-
 .../RISCV/vplan-vp-select-intrinsics.ll            |   4 +-
 .../LoopVectorize/outer_loop_scalable.ll           |   6 +-
 .../scalable-first-order-recurrence.ll             | 134 ++++-----
 .../LoopVectorize/scalable-inductions.ll           |  22 +-
 .../LoopVectorize/scalable-iv-outside-user.ll      |  10 +-
 .../Transforms/LoopVectorize/scalable-lifetime.ll  |  12 +-
 .../scalable-loop-unpredicated-body-scalar-tail.ll |  12 +-
 .../LoopVectorize/scalable-reduction-inloop.ll     |   8 +-
 .../LoopVectorize/scalable-trunc-min-bitwidth.ll   |  12 +-
 .../LoopVectorize/vectorize-force-tail-with-evl.ll |   6 +-
 llvm/test/Transforms/MemCpyOpt/vscale-crashes.ll   |   4 +-
 .../AArch64/sve-interleave-vectorization.ll        |   6 +-
 .../PreISelIntrinsicLowering/AArch64/expand-exp.ll |   2 +-
 134 files changed, 1929 insertions(+), 1927 deletions(-)

diff --git a/clang/test/CodeGen/builtin_vectorelements.c b/clang/test/CodeGen/builtin_vectorelements.c
index b0ff6f83b1e4..45f7a3c34562 100644
--- a/clang/test/CodeGen/builtin_vectorelements.c
+++ b/clang/test/CodeGen/builtin_vectorelements.c
@@ -85,7 +85,7 @@ int test_builtin_vectorelements_neon64x1() {
 long test_builtin_vectorelements_sve32() {
   // SVE: i64 @test_builtin_vectorelements_sve32(
   // SVE: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
-  // SVE: [[RES:%.+]] = mul i64 [[VSCALE]], 4
+  // SVE: [[RES:%.+]] = mul nuw i64 [[VSCALE]], 4
   // SVE: ret i64 [[RES]]
   return __builtin_vectorelements(svuint32_t);
 }
@@ -93,7 +93,7 @@ long test_builtin_vectorelements_sve32() {
 long test_builtin_vectorelements_sve8() {
   // SVE: i64 @test_builtin_vectorelements_sve8(
   // SVE: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
-  // SVE: [[RES:%.+]] = mul i64 [[VSCALE]], 16
+  // SVE: [[RES:%.+]] = mul nuw i64 [[VSCALE]], 16
   // SVE: ret i64 [[RES]]
   return __builtin_vectorelements(svuint8_t);
 }
@@ -105,7 +105,7 @@ long test_builtin_vectorelements_sve8() {
 long test_builtin_vectorelements_riscv8() {
   // RISCV: i64 @test_builtin_vectorelements_riscv8(
   // RISCV: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
-  // RISCV: [[RES:%.+]] = mul i64 [[VSCALE]], 8
+  // RISCV: [[RES:%.+]] = mul nuw i64 [[VSCALE]], 8
   // RISCV: ret i64 [[RES]]
   return __builtin_vectorelements(vuint8m1_t);
 }
@@ -120,7 +120,7 @@ long test_builtin_vectorelements_riscv64() {
 long test_builtin_vectorelements_riscv32m2() {
   // RISCV: i64 @test_builtin_vectorelements_riscv32m2(
   // RISCV: [[VSCALE:%.+]] = call i64 @llvm.vscale.i64()
-  // RISCV: [[RES:%.+]] = mul i64 [[VSCALE]], 4
+  // RISCV: [[RES:%.+]] = mul nuw i64 [[VSCALE]], 4
   // RISCV: ret i64 [[RES]]
   return __builtin_vectorelements(vuint32m2_t);
 }
diff --git a/llvm/include/llvm/IR/IRBuilder.h b/llvm/include/llvm/IR/IRBuilder.h
index 59295089d6e9..8fe610835dca 100644
--- a/llvm/include/llvm/IR/IRBuilder.h
+++ b/llvm/include/llvm/IR/IRBuilder.h
@@ -965,11 +965,13 @@ public:
   }
 
   /// Create an expression which evaluates to the number of elements in \p EC
-  /// at runtime.
+  /// at runtime. This can result in poison if type \p Ty is not big enough to
+  /// hold the value.
   LLVM_ABI Value *CreateElementCount(Type *Ty, ElementCount EC);
 
   /// Create an expression which evaluates to the number of units in \p Size
-  /// at runtime.  This works for both units of bits and bytes.
+  /// at runtime. This works for both units of bits and bytes. This can result
+  /// in poison if type \p Ty is not big enough to hold the value.
   LLVM_ABI Value *CreateTypeSize(Type *Ty, TypeSize Size);
 
   /// Creates a vector of type \p DstType with the linear sequence <0, 1, ...>
diff --git a/llvm/lib/IR/IRBuilder.cpp b/llvm/lib/IR/IRBuilder.cpp
index 0a8b26b5f3d8..beefd5e7d7ee 100644
--- a/llvm/lib/IR/IRBuilder.cpp
+++ b/llvm/lib/IR/IRBuilder.cpp
@@ -117,7 +117,7 @@ static Value *CreateVScaleMultiple(IRBuilderBase &B, Type *Ty, uint64_t Scale) {
   if (Scale == 1)
     return VScale;
 
-  return B.CreateMul(VScale, ConstantInt::get(Ty, Scale));
+  return B.CreateNUWMul(VScale, ConstantInt::get(Ty, Scale));
 }
 
 Value *IRBuilderBase::CreateElementCount(Type *Ty, ElementCount EC) {
diff --git a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
index 436aadbc25de..b8a5be583499 100644
--- a/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
+++ b/llvm/test/Analysis/ValueTracking/phi-known-bits.ll
@@ -936,7 +936,7 @@ define i1 @recursiveGEP_withPtrSub_scalableGEP(ptr %val1) {
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TEST_0_I]] = getelementptr i8, ptr [[A_PN_I]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[TEST_0_I]], align 1
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP2]], 0
@@ -970,7 +970,7 @@ define i1 @recursiveGEP_withPtrSub_scalableGEP_inbounds(ptr %val1) {
 ; CHECK:       while.cond.i:
 ; CHECK-NEXT:    [[A_PN_I:%.*]] = phi ptr [ [[TEST_0_I:%.*]], [[WHILE_COND_I]] ], [ [[VAL1:%.*]], [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TEST_0_I]] = getelementptr inbounds i8, ptr [[A_PN_I]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = load i8, ptr [[TEST_0_I]], align 1
 ; CHECK-NEXT:    [[CMP3_NOT_I:%.*]] = icmp eq i8 [[TMP2]], 0
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-masked-load-store.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-masked-load-store.ll
index a8da5a3740e5..afa3d0966b55 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-masked-load-store.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-masked-load-store.ll
@@ -315,7 +315,7 @@ declare void @llvm.masked.store.nxv4f32.p0(<vscale x 4 x float>, ptr, i32, <vsca
 define <vscale x 4 x float> @scalable.load.nxv4f32(ptr %p, <vscale x 4 x i1> %mask) sanitize_address {
 ; CHECK-LABEL: @scalable.load.nxv4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[TMP7:%.*]] ]
@@ -345,7 +345,7 @@ define <vscale x 4 x float> @scalable.load.nxv4f32(ptr %p, <vscale x 4 x i1> %ma
 define void @scalable.store.nxv4f32(ptr %p, <vscale x 4 x float> %arg, <vscale x 4 x i1> %mask) sanitize_address {
 ; CHECK-LABEL: @scalable.store.nxv4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[TMP7:%.*]] ]
@@ -379,7 +379,7 @@ declare void @llvm.masked.scatter.nxv4f32.nxv4p0(<vscale x 4 x float>, <vscale x
 define <vscale x 4 x float> @scalable.gather.nxv4f32(<vscale x 4 x ptr> %vp, <vscale x 4 x i1> %mask, i32 %evl) sanitize_address {
 ; CHECK-LABEL: @scalable.gather.nxv4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[TMP7:%.*]] ]
@@ -409,7 +409,7 @@ define <vscale x 4 x float> @scalable.gather.nxv4f32(<vscale x 4 x ptr> %vp, <vs
 define void @scalable.scatter.nxv4f32(<vscale x 4 x float> %val, <vscale x 4 x ptr> %vp, <vscale x 4 x i1> %mask, i32 %evl) sanitize_address {
 ; CHECK-LABEL: @scalable.scatter.nxv4f32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[IV_NEXT:%.*]], [[TMP7:%.*]] ]
@@ -447,7 +447,7 @@ define <vscale x 4 x float> @scalable.expandload.nxv4f32(ptr align 4 %p, <vscale
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]]
 ; CHECK:       4:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP6]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
@@ -485,7 +485,7 @@ define void @scalable.compressstore.nxv4f32(ptr align 4 %p, <vscale x 4 x float>
 ; CHECK-NEXT:    br i1 [[TMP3]], label [[TMP4:%.*]], label [[TMP13:%.*]]
 ; CHECK:       4:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP6]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
diff --git a/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll b/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll
index d22671aa84f8..f53aa44c8c90 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/asan-vp-load-store.ll
@@ -255,7 +255,7 @@ define <vscale x 4 x float> @scalable.load.nxv4f32(ptr align 4 %p, <vscale x 4 x
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
@@ -292,7 +292,7 @@ define void @scalable.store.nxv4f32(ptr align 4 %p, <vscale x 4 x float> %arg, <
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
@@ -333,7 +333,7 @@ define <vscale x 4 x float> @scalable.strided.load.nxv4f32(ptr align 4 %p, i32 %
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -372,7 +372,7 @@ define void @scalable.strided.store.nxv4f32(<vscale x 4 x float> %arg, ptr align
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    [[TMP7:%.*]] = zext i32 [[STRIDE:%.*]] to i64
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -412,7 +412,7 @@ define <vscale x 4 x float> @scalable.strided.load.nxv4f32.align(ptr align 4 %p,
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
@@ -454,7 +454,7 @@ define <vscale x 4 x float> @scalable.gather.nxv4f32(<vscale x 4 x ptr> %vp, <vs
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
@@ -491,7 +491,7 @@ define void @scalable.scatter.nxv4f32(<vscale x 4 x float> %arg, <vscale x 4 x p
 ; CHECK:       2:
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext i32 [[EVL]] to i64
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP3]], i64 [[TMP5]])
 ; CHECK-NEXT:    br label [[DOTSPLIT:%.*]]
 ; CHECK:       .split:
diff --git a/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll b/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
index 120a4b235f85..373cb8a536a2 100644
--- a/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
+++ b/llvm/test/Instrumentation/AddressSanitizer/vector-load-store.ll
@@ -402,7 +402,7 @@ define void @store.v2i32.align8(ptr %p) sanitize_address {
 define void @load.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @load.nxv1i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 32
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -445,7 +445,7 @@ define void @load.nxv1i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @load.nxv1i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 32
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 32
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
@@ -459,7 +459,7 @@ define void @load.nxv1i32(ptr %p) sanitize_address {
 define void @load.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @load.nxv2i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -502,7 +502,7 @@ define void @load.nxv2i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @load.nxv2i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
@@ -516,7 +516,7 @@ define void @load.nxv2i32(ptr %p) sanitize_address {
 define void @load.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @load.nxv4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 128
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -559,7 +559,7 @@ define void @load.nxv4i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @load.nxv4i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 128
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 128
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
@@ -573,7 +573,7 @@ define void @load.nxv4i32(ptr %p) sanitize_address {
 define void @load.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @load.nxv8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 256
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 256
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -616,7 +616,7 @@ define void @load.nxv8i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @load.nxv8i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 256
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 256
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
@@ -630,7 +630,7 @@ define void @load.nxv8i32(ptr %p) sanitize_address {
 define void @load.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @load.nxv16i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 512
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 512
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -673,7 +673,7 @@ define void @load.nxv16i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @load.nxv16i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 512
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 512
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
@@ -688,7 +688,7 @@ define void @load.nxv16i32(ptr %p) sanitize_address {
 define void @store.nxv1i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @store.nxv1i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 32
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -731,7 +731,7 @@ define void @store.nxv1i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @store.nxv1i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 32
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 32
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_storeN(i64 [[TMP4]], i64 [[TMP3]])
@@ -745,7 +745,7 @@ define void @store.nxv1i32(ptr %p) sanitize_address {
 define void @store.nxv2i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @store.nxv2i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -788,7 +788,7 @@ define void @store.nxv2i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @store.nxv2i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_storeN(i64 [[TMP4]], i64 [[TMP3]])
@@ -802,7 +802,7 @@ define void @store.nxv2i32(ptr %p) sanitize_address {
 define void @store.nxv4i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @store.nxv4i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 128
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 128
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -845,7 +845,7 @@ define void @store.nxv4i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @store.nxv4i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 128
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 128
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_storeN(i64 [[TMP4]], i64 [[TMP3]])
@@ -859,7 +859,7 @@ define void @store.nxv4i32(ptr %p) sanitize_address {
 define void @store.nxv8i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @store.nxv8i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 256
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 256
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -902,7 +902,7 @@ define void @store.nxv8i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @store.nxv8i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 256
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 256
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_storeN(i64 [[TMP4]], i64 [[TMP3]])
@@ -916,7 +916,7 @@ define void @store.nxv8i32(ptr %p) sanitize_address {
 define void @store.nxv16i32(ptr %p) sanitize_address {
 ; CHECK-LABEL: @store.nxv16i32(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 512
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 512
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -959,7 +959,7 @@ define void @store.nxv16i32(ptr %p) sanitize_address {
 ;
 ; CALLS-LABEL: @store.nxv16i32(
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 512
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 512
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CALLS-NEXT:    call void @__asan_storeN(i64 [[TMP4]], i64 [[TMP3]])
@@ -977,7 +977,7 @@ define <vscale x 2 x i32> @local_alloca() sanitize_address {
 ; CHECK-NEXT:    [[A:%.*]] = alloca <vscale x 2 x i32>, align 8
 ; CHECK-NEXT:    call void @clobber(ptr [[A]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CHECK-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CHECK-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP3]], 1
@@ -1022,7 +1022,7 @@ define <vscale x 2 x i32> @local_alloca() sanitize_address {
 ; CALLS-NEXT:    [[A:%.*]] = alloca <vscale x 2 x i32>, align 8
 ; CALLS-NEXT:    call void @clobber(ptr [[A]])
 ; CALLS-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CALLS-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 64
+; CALLS-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 64
 ; CALLS-NEXT:    [[TMP3:%.*]] = lshr i64 [[TMP2]], 3
 ; CALLS-NEXT:    [[TMP4:%.*]] = ptrtoint ptr [[A]] to i64
 ; CALLS-NEXT:    call void @__asan_loadN(i64 [[TMP4]], i64 [[TMP3]])
diff --git a/llvm/test/Instrumentation/BoundsChecking/simple.ll b/llvm/test/Instrumentation/BoundsChecking/simple.ll
index 5d8f76753e0c..ab2f9f66c82f 100644
--- a/llvm/test/Instrumentation/BoundsChecking/simple.ll
+++ b/llvm/test/Instrumentation/BoundsChecking/simple.ll
@@ -470,7 +470,7 @@ define <vscale x 1 x i32> @load_scalable_vector(i64 %y) nounwind {
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i64 [[Y]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4, !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 256, [[TMP2]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp ult i64 256, [[TMP2]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]], !nosanitize [[META0]]
@@ -492,16 +492,16 @@ define <vscale x 1 x i32> @load_scalable_vector(i64 %y) nounwind {
 define void @scalable_alloca(i64 %y) nounwind {
 ; CHECK-LABEL: @scalable_alloca(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[TMP4:%.*]] = alloca <vscale x 4 x i16>, i32 5, align 8
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds <vscale x 4 x i16>, ptr [[TMP4]], i64 [[Y]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8, !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]], !nosanitize [[META0]]
@@ -525,16 +525,16 @@ define void @scalable_alloca(i64 %y) nounwind {
 define void @scalable_alloca2(i64 %y) nounwind {
 ; CHECK-LABEL: @scalable_alloca2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 32
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 32
 ; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = alloca <vscale x 4 x i64>, align 32
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 32
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
 ; CHECK-NEXT:    [[DOTIDX:%.*]] = mul i64 [[Y:%.*]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i64 0, [[DOTIDX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds <vscale x 4 x i64>, ptr [[TMP4]], i64 [[Y]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64(), !nosanitize [[META0]]
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 32, !nosanitize [[META0]]
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 32, !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = icmp ult i64 [[TMP3]], [[TMP7]], !nosanitize [[META0]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ult i64 [[TMP11]], [[TMP10]], !nosanitize [[META0]]
diff --git a/llvm/test/Instrumentation/HWAddressSanitizer/vector-load-store.ll b/llvm/test/Instrumentation/HWAddressSanitizer/vector-load-store.ll
index 5312c7cc7336..e3794e37f8b2 100644
--- a/llvm/test/Instrumentation/HWAddressSanitizer/vector-load-store.ll
+++ b/llvm/test/Instrumentation/HWAddressSanitizer/vector-load-store.ll
@@ -125,7 +125,7 @@ define void @load.nxv1i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 1 x i32>, ptr [[P]], align 4
@@ -140,7 +140,7 @@ define void @load.nxv2i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 64
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 64
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 2 x i32>, ptr [[P]], align 8
@@ -155,7 +155,7 @@ define void @load.nxv4i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 128
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 4 x i32>, ptr [[P]], align 16
@@ -170,7 +170,7 @@ define void @load.nxv8i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 256
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 256
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 8 x i32>, ptr [[P]], align 32
@@ -185,7 +185,7 @@ define void @load.nxv16i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 512
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 512
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    [[TMP5:%.*]] = load <vscale x 16 x i32>, ptr [[P]], align 64
@@ -201,7 +201,7 @@ define void @store.nxv1i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    store <vscale x 1 x i32> zeroinitializer, ptr [[P]], align 4
@@ -216,7 +216,7 @@ define void @store.nxv2i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 64
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 64
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    store <vscale x 2 x i32> zeroinitializer, ptr [[P]], align 8
@@ -231,7 +231,7 @@ define void @store.nxv4i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 128
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 128
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    store <vscale x 4 x i32> zeroinitializer, ptr [[P]], align 16
@@ -246,7 +246,7 @@ define void @store.nxv8i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 256
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 256
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    store <vscale x 8 x i32> zeroinitializer, ptr [[P]], align 32
@@ -261,7 +261,7 @@ define void @store.nxv16i32(ptr %p) sanitize_hwaddress {
 ; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
 ; CHECK-NEXT:    [[TMP1:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 512
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 512
 ; CHECK-NEXT:    [[TMP4:%.*]] = udiv i64 [[TMP3]], 8
 ; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP1]], i64 [[TMP4]])
 ; CHECK-NEXT:    store <vscale x 16 x i32> zeroinitializer, ptr [[P]], align 64
diff --git a/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll b/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
index 0149b4f05559..d01974016f6c 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/vector-load-store.ll
@@ -672,7 +672,7 @@ define void @store.nxv1i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1:![0-9]+]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; ORIGINS-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; ORIGINS-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 3
 ; ORIGINS-NEXT:    [[TMP11:%.*]] = udiv i64 [[TMP10]], 4
 ; ORIGINS-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -732,7 +732,7 @@ define void @store.nxv2i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; ORIGINS-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; ORIGINS-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 3
 ; ORIGINS-NEXT:    [[TMP11:%.*]] = udiv i64 [[TMP10]], 4
 ; ORIGINS-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -792,7 +792,7 @@ define void @store.nxv4i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; ORIGINS-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; ORIGINS-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 3
 ; ORIGINS-NEXT:    [[TMP11:%.*]] = udiv i64 [[TMP10]], 4
 ; ORIGINS-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -852,7 +852,7 @@ define void @store.nxv8i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 32
+; ORIGINS-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 32
 ; ORIGINS-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 3
 ; ORIGINS-NEXT:    [[TMP11:%.*]] = udiv i64 [[TMP10]], 4
 ; ORIGINS-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -912,7 +912,7 @@ define void @store.nxv16i32(ptr %p) sanitize_memory {
 ; ORIGINS-NEXT:    br i1 [[_MSCMP]], label [[TMP7:%.*]], label [[TMP13:%.*]], !prof [[PROF1]]
 ; ORIGINS:       7:
 ; ORIGINS-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGINS-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 64
+; ORIGINS-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 64
 ; ORIGINS-NEXT:    [[TMP10:%.*]] = add i64 [[TMP9]], 3
 ; ORIGINS-NEXT:    [[TMP11:%.*]] = udiv i64 [[TMP10]], 4
 ; ORIGINS-NEXT:    br label [[DOTSPLIT:%.*]]
diff --git a/llvm/test/Instrumentation/MemorySanitizer/vscale.ll b/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
index 61114af7ce90..0c0b393667bf 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/vscale.ll
@@ -44,7 +44,7 @@ define void @test_load_store_i32(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN:       14:
 ; ORIGIN-NEXT:    [[TMP15:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP7]])
 ; ORIGIN-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 16
+; ORIGIN-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 16
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 3
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = udiv i64 [[TMP18]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -125,7 +125,7 @@ define void @test_load_store_add_int(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN:       25:
 ; ORIGIN-NEXT:    [[TMP26:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP14]])
 ; ORIGIN-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 64
+; ORIGIN-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 64
 ; ORIGIN-NEXT:    [[TMP29:%.*]] = add i64 [[TMP28]], 3
 ; ORIGIN-NEXT:    [[TMP30:%.*]] = udiv i64 [[TMP29]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -188,7 +188,7 @@ define void @test_load_store_float(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN:       14:
 ; ORIGIN-NEXT:    [[TMP15:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP7]])
 ; ORIGIN-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 16
+; ORIGIN-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 16
 ; ORIGIN-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], 3
 ; ORIGIN-NEXT:    [[TMP19:%.*]] = udiv i64 [[TMP18]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -269,7 +269,7 @@ define void @test_load_store_add_float(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN:       25:
 ; ORIGIN-NEXT:    [[TMP26:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP14]])
 ; ORIGIN-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
+; ORIGIN-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
 ; ORIGIN-NEXT:    [[TMP29:%.*]] = add i64 [[TMP28]], 3
 ; ORIGIN-NEXT:    [[TMP30:%.*]] = udiv i64 [[TMP29]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -363,7 +363,7 @@ define void @test_ret(ptr %a, ptr %b) sanitize_memory {
 ; ORIGIN:       11:
 ; ORIGIN-NEXT:    [[TMP12:%.*]] = call i32 @__msan_chain_origin(i32 [[TMP4]])
 ; ORIGIN-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 8
+; ORIGIN-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
 ; ORIGIN-NEXT:    [[TMP15:%.*]] = add i64 [[TMP14]], 3
 ; ORIGIN-NEXT:    [[TMP16:%.*]] = udiv i64 [[TMP15]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -411,7 +411,7 @@ define void @fn_param(<vscale x 2 x float> %a, ptr %b) sanitize_memory {
 ; ORIGIN:       7:
 ; ORIGIN-NEXT:    [[TMP8:%.*]] = call i32 @__msan_chain_origin(i32 0)
 ; ORIGIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; ORIGIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; ORIGIN-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], 3
 ; ORIGIN-NEXT:    [[TMP12:%.*]] = udiv i64 [[TMP11]], 4
 ; ORIGIN-NEXT:    br label [[DOTSPLIT:%.*]]
@@ -490,7 +490,7 @@ define void @test_alloca1() sanitize_memory {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[X:%.*]] = alloca <vscale x 64 x i1>, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[X]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
 ; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
@@ -503,7 +503,7 @@ define void @test_alloca1() sanitize_memory {
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[X:%.*]] = alloca <vscale x 64 x i1>, align 4
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; ORIGIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[X]] to i64
 ; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
 ; ORIGIN-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
@@ -526,7 +526,7 @@ define void @test_alloca2() sanitize_memory {
 ; CHECK-NEXT:    call void @llvm.donothing()
 ; CHECK-NEXT:    [[X:%.*]] = alloca <vscale x 64 x double>, align 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 512
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 512
 ; CHECK-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[X]] to i64
 ; CHECK-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
 ; CHECK-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
@@ -539,7 +539,7 @@ define void @test_alloca2() sanitize_memory {
 ; ORIGIN-NEXT:    call void @llvm.donothing()
 ; ORIGIN-NEXT:    [[X:%.*]] = alloca <vscale x 64 x double>, align 4
 ; ORIGIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ORIGIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 512
+; ORIGIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 512
 ; ORIGIN-NEXT:    [[TMP2:%.*]] = ptrtoint ptr [[X]] to i64
 ; ORIGIN-NEXT:    [[TMP3:%.*]] = xor i64 [[TMP2]], 87960930222080
 ; ORIGIN-NEXT:    [[TMP4:%.*]] = inttoptr i64 [[TMP3]] to ptr
diff --git a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
index 46ca99f4bb27..0722e4cfcddd 100644
--- a/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
+++ b/llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-opts-counting-elems.ll
@@ -52,7 +52,7 @@ define i64 @cntb_mul4() {
 define i64 @cntb_all() {
 ; CHECK-LABEL: @cntb_all(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    ret i64 [[OUT]]
 ;
   %out = call i64 @llvm.aarch64.sve.cntb(i32 31)
@@ -110,7 +110,7 @@ define i64 @cnth_mul4() {
 define i64 @cnth_all() {
 ; CHECK-LABEL: @cnth_all(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 3
+; CHECK-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-NEXT:    ret i64 [[OUT]]
 ;
   %out = call i64 @llvm.aarch64.sve.cnth(i32 31)
@@ -170,7 +170,7 @@ define i64 @cntw_mul4() {
 define i64 @cntw_all() {
 ; CHECK-LABEL: @cntw_all(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    ret i64 [[OUT]]
 ;
   %out = call i64 @llvm.aarch64.sve.cntw(i32 31)
@@ -232,7 +232,7 @@ define i64 @cntd_mul4() {
 define i64 @cntd_all() {
 ; CHECK-LABEL: @cntd_all(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[OUT:%.*]] = shl i64 [[TMP1]], 1
+; CHECK-NEXT:    [[OUT:%.*]] = shl nuw i64 [[TMP1]], 1
 ; CHECK-NEXT:    ret i64 [[OUT]]
 ;
   %out = call i64 @llvm.aarch64.sve.cntd(i32 31)
diff --git a/llvm/test/Transforms/InstCombine/gep-vector.ll b/llvm/test/Transforms/InstCombine/gep-vector.ll
index 27624f790c4c..5546cb36d2f5 100644
--- a/llvm/test/Transforms/InstCombine/gep-vector.ll
+++ b/llvm/test/Transforms/InstCombine/gep-vector.ll
@@ -128,7 +128,7 @@ define ptr addrspace(3) @inbounds_bitcast_vec_to_array_addrspace_matching_alloc_
 define ptr @test_accumulate_constant_offset_vscale_nonzero(<vscale x 16 x i1> %pg, ptr %base) {
 ; CHECK-LABEL: @test_accumulate_constant_offset_vscale_nonzero(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[BASE:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[TMP3]], i64 4
 ; CHECK-NEXT:    ret ptr [[GEP]]
diff --git a/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll b/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
index 4c8c56a9262e..a92e0c263d35 100644
--- a/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
+++ b/llvm/test/Transforms/InstCombine/gepofconstgepi8.ll
@@ -282,7 +282,7 @@ define ptr @test_scalable(ptr %base, i64 %a) {
 ; CHECK-NEXT:    [[P1:%.*]] = getelementptr i8, ptr [[BASE]], i64 -4
 ; CHECK-NEXT:    [[INDEX:%.*]] = add i64 [[A]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[P2_IDX:%.*]] = mul i64 [[INDEX]], [[TMP1]]
 ; CHECK-NEXT:    [[P2:%.*]] = getelementptr i8, ptr [[P1]], i64 [[P2_IDX]]
 ; CHECK-NEXT:    ret ptr [[P2]]
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index 61236df80bfa..e78d70058c14 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -266,7 +266,7 @@ define <2 x i1> @test13_fixed_scalable(i64 %X, ptr %P, <2 x i64> %y) nounwind {
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[DOTSPLATINSERT]], <i64 3, i64 0>
 ; CHECK-NEXT:    [[A_IDX:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <2 x i64> [[DOTSPLATINSERT1]], <2 x i64> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[B_IDX:%.*]] = mul nsw <2 x i64> [[Y:%.*]], [[DOTSPLAT]]
@@ -285,7 +285,7 @@ define <vscale x 2 x i1> @test13_scalable_scalable(i64 %X, ptr %P, <vscale x 2 x
 ; CHECK-NEXT:    [[TMP3:%.*]] = shl nsw <vscale x 2 x i64> [[DOTSPLATINSERT]], splat (i64 3)
 ; CHECK-NEXT:    [[A_IDX:%.*]] = shufflevector <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[DOTSPLATINSERT1:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP2]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT2:%.*]] = shufflevector <vscale x 2 x i64> [[DOTSPLATINSERT1]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[B_IDX:%.*]] = mul nsw <vscale x 2 x i64> [[Y:%.*]], [[DOTSPLAT2]]
diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll
index 7f8f1ae73948..260462896c39 100644
--- a/llvm/test/Transforms/InstCombine/icmp-gep.ll
+++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll
@@ -623,10 +623,10 @@ define i1 @test_scalable_xc(ptr %x) {
 define i1 @test_scalable_xy(ptr %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test_scalable_xy(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[GEP1_IDX:%.*]] = mul nsw i64 [[I:%.*]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[GEP2_IDX:%.*]] = mul nsw i64 [[J:%.*]], [[TMP4]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i64 [[GEP2_IDX]], [[GEP1_IDX]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
@@ -642,10 +642,10 @@ define i1 @test_scalable_xy(ptr %foo, i64 %i, i64 %j) {
 define i1 @test_scalable_ij(ptr %foo, i64 %i, i64 %j) {
 ; CHECK-LABEL: @test_scalable_ij(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[GEP1_IDX:%.*]] = mul nsw i64 [[I:%.*]], [[TMP2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[GEP2_IDX:%.*]] = mul nsw i64 [[J:%.*]], [[TMP4]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i64 [[GEP1_IDX]], [[GEP2_IDX]]
 ; CHECK-NEXT:    ret i1 [[CMP]]
diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
index 0f8b06e12421..d9f022442a02 100644
--- a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
+++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll
@@ -326,7 +326,7 @@ define void @scatter_nxv4i16_no_uniform_vals_uniform_ptrs_all_active_mask(ptr %d
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[SRC:%.*]], align 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i32 [[TMP1]], -1
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP2]]
 ; CHECK-NEXT:    store i16 [[TMP3]], ptr [[DST:%.*]], align 2
diff --git a/llvm/test/Transforms/InstCombine/opaque-ptr.ll b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
index be734243d14a..99d1fa032db1 100644
--- a/llvm/test/Transforms/InstCombine/opaque-ptr.ll
+++ b/llvm/test/Transforms/InstCombine/opaque-ptr.ll
@@ -294,7 +294,7 @@ define ptr @geps_combinable_different_elem_type_extra_use2(ptr %a, i64 %idx) {
 define ptr @geps_combinable_scalable(ptr %a, i64 %idx) {
 ; CHECK-LABEL: @geps_combinable_scalable(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw i8, ptr [[A2]], i64 4
 ; CHECK-NEXT:    ret ptr [[A3]]
@@ -307,7 +307,7 @@ define ptr @geps_combinable_scalable(ptr %a, i64 %idx) {
 define ptr @geps_combinable_scalable_vector_array(ptr %a, i64 %idx) {
 ; CHECK-LABEL: @geps_combinable_scalable_vector_array(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 5
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 5
 ; CHECK-NEXT:    [[A2:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[A3:%.*]] = getelementptr inbounds nuw i8, ptr [[A2]], i64 4
 ; CHECK-NEXT:    ret ptr [[A3]]
diff --git a/llvm/test/Transforms/InstCombine/scalable-vector-array.ll b/llvm/test/Transforms/InstCombine/scalable-vector-array.ll
index 20e9f2d99dd9..290de9633fc8 100644
--- a/llvm/test/Transforms/InstCombine/scalable-vector-array.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-vector-array.ll
@@ -5,7 +5,7 @@ define <vscale x 4 x i32> @load(ptr %x) {
 ; CHECK-LABEL: define <vscale x 4 x i32> @load
 ; CHECK-SAME: (ptr [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[A_ELT1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[A_UNPACK2:%.*]] = load <vscale x 4 x i32>, ptr [[A_ELT1]], align 16
 ; CHECK-NEXT:    ret <vscale x 4 x i32> [[A_UNPACK2]]
@@ -20,7 +20,7 @@ define void @store(ptr %x, <vscale x 4 x i32> %y, <vscale x 4 x i32> %z) {
 ; CHECK-SAME: (ptr [[X:%.*]], <vscale x 4 x i32> [[Y:%.*]], <vscale x 4 x i32> [[Z:%.*]]) {
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[Y]], ptr [[X]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[X_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP2]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[Z]], ptr [[X_REPACK1]], align 16
 ; CHECK-NEXT:    ret void
diff --git a/llvm/test/Transforms/InstCombine/scalable-vector-struct.ll b/llvm/test/Transforms/InstCombine/scalable-vector-struct.ll
index a8790b579d75..9a0a6ae6324e 100644
--- a/llvm/test/Transforms/InstCombine/scalable-vector-struct.ll
+++ b/llvm/test/Transforms/InstCombine/scalable-vector-struct.ll
@@ -7,7 +7,7 @@ define <vscale x 1 x i32> @load(ptr %x) {
 ; CHECK-LABEL: define <vscale x 1 x i32> @load
 ; CHECK-SAME: (ptr [[X:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[A_ELT1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[A_UNPACK2:%.*]] = load <vscale x 1 x i32>, ptr [[A_ELT1]], align 4
 ; CHECK-NEXT:    ret <vscale x 1 x i32> [[A_UNPACK2]]
@@ -22,7 +22,7 @@ define void @store(ptr %x, <vscale x 1 x i32> %y, <vscale x 1 x i32> %z) {
 ; CHECK-SAME: (ptr [[X:%.*]], <vscale x 1 x i32> [[Y:%.*]], <vscale x 1 x i32> [[Z:%.*]]) {
 ; CHECK-NEXT:    store <vscale x 1 x i32> [[Y]], ptr [[X]], align 4
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[X_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[X]], i64 [[TMP2]]
 ; CHECK-NEXT:    store <vscale x 1 x i32> [[Z]], ptr [[X_REPACK1]], align 4
 ; CHECK-NEXT:    ret void
@@ -40,7 +40,7 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @split_load(ptr %p) nounwind {
 ; CHECK-NEXT:    [[R_UNPACK:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> [[R_UNPACK]], 0
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[R_ELT1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[R_UNPACK2:%.*]] = load <vscale x 16 x i8>, ptr [[R_ELT1]], align 16
 ; CHECK-NEXT:    [[R3:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP0]], <vscale x 16 x i8> [[R_UNPACK2]], 1
@@ -71,7 +71,7 @@ define void @split_store({<vscale x 4 x i32>, <vscale x 4 x i32>} %x, ptr %p) no
 ; CHECK-NEXT:    [[X_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 0
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT]], ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[X_ELT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 1
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT2]], ptr [[P_REPACK1]], align 16
@@ -123,14 +123,14 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @check_nxv16i8_nxv4i32({<vscale
 ; CHECK-NEXT:    [[X_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 0
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT]], ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[X_ELT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 1
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT2]], ptr [[P_REPACK1]], align 16
 ; CHECK-NEXT:    [[R_UNPACK:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> [[R_UNPACK]], 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[R_ELT3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[R_UNPACK4:%.*]] = load <vscale x 16 x i8>, ptr [[R_ELT3]], align 16
 ; CHECK-NEXT:    [[R5:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], <vscale x 16 x i8> [[R_UNPACK4]], 1
@@ -150,14 +150,14 @@ define {<vscale x 16 x i8>, <vscale x 16 x i8>} @alloca_nxv16i8_nxv4i32({<vscale
 ; CHECK-NEXT:    [[X_ELT:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 0
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT]], ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[P_REPACK1:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP1]]
 ; CHECK-NEXT:    [[X_ELT2:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[X]], 1
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[X_ELT2]], ptr [[P_REPACK1]], align 16
 ; CHECK-NEXT:    [[R_UNPACK:%.*]] = load <vscale x 16 x i8>, ptr [[P]], align 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } poison, <vscale x 16 x i8> [[R_UNPACK]], 0
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = shl i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = shl nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[R_ELT3:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[R_UNPACK4:%.*]] = load <vscale x 16 x i8>, ptr [[R_ELT3]], align 16
 ; CHECK-NEXT:    [[R5:%.*]] = insertvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], <vscale x 16 x i8> [[R_UNPACK4]], 1
diff --git a/llvm/test/Transforms/InstCombine/sub-gep.ll b/llvm/test/Transforms/InstCombine/sub-gep.ll
index 375be8a3d69c..45a30350aafe 100644
--- a/llvm/test/Transforms/InstCombine/sub-gep.ll
+++ b/llvm/test/Transforms/InstCombine/sub-gep.ll
@@ -703,7 +703,7 @@ define i64 @sub_scalable(ptr noundef %val1) {
 ; CHECK-LABEL: @sub_scalable(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    ret i64 [[TMP1]]
 ;
 entry:
@@ -718,7 +718,7 @@ define i64 @sub_scalable2(ptr noundef %val1) {
 ; CHECK-LABEL: @sub_scalable2(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[GEP2_IDX:%.*]] = shl i64 [[TMP2]], 5
 ; CHECK-NEXT:    [[GEPDIFF:%.*]] = sub i64 [[TMP1]], [[GEP2_IDX]]
@@ -750,7 +750,7 @@ define i64 @nullptrtoint_scalable_x(i64 %x) {
 ; CHECK-LABEL: @nullptrtoint_scalable_x(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[PTR_IDX:%.*]] = mul nsw i64 [[X:%.*]], [[TMP1]]
 ; CHECK-NEXT:    ret i64 [[PTR_IDX]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/vscale_gep.ll b/llvm/test/Transforms/InstCombine/vscale_gep.ll
index 371ee71e45f2..84019e613d23 100644
--- a/llvm/test/Transforms/InstCombine/vscale_gep.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_gep.ll
@@ -15,7 +15,7 @@ define <vscale x 2 x ptr> @gep_index_type_is_scalable(ptr %p) {
 define ptr @gep_num_of_indices_1(ptr %p) {
 ; CHECK-LABEL: @gep_num_of_indices_1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    ret ptr [[GEP]]
 ;
@@ -28,7 +28,7 @@ define void @gep_bitcast(ptr %p) {
 ; CHECK-LABEL: @gep_bitcast(
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[P:%.*]], align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[GEP2:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP2]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[GEP2]], align 16
 ; CHECK-NEXT:    ret void
@@ -59,7 +59,7 @@ define i32 @gep_alloca_inbounds_vscale_nonzero() {
 ; CHECK-LABEL: @gep_alloca_inbounds_vscale_nonzero(
 ; CHECK-NEXT:    [[A:%.*]] = alloca <vscale x 4 x i32>, align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = shl i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[TMP]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
index 24c703ae42f0..95f3eb7b21f4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/clamped-trip-count.ll
@@ -8,13 +8,13 @@ define void @clamped_tc_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range(1,1
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 8, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 8)
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
@@ -92,13 +92,13 @@ define void @clamped_tc_max_8(ptr nocapture %dst, i32 %n, i64 %val) vscale_range
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[WIDE_TRIP_COUNT]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[VAL]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
index 976f95ff4f0b..5f72fa4b4e8e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/conditional-branches-cost.ll
@@ -706,15 +706,15 @@ define void @multiple_exit_conditions(ptr %src, ptr noalias %dst) #1 {
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; PRED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 257, [[TMP2]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 257, [[TMP7]]
 ; PRED-NEXT:    [[TMP9:%.*]] = icmp ugt i64 257, [[TMP7]]
 ; PRED-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
index d59607711b5b..d42be20ea1e7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/divs-with-scalable-vfs.ll
@@ -8,7 +8,7 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[CONV61:%.*]] = zext i32 [[X]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
@@ -21,11 +21,11 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP9]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = sdiv i64 [[M]], [[CONV6]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = trunc i64 [[TMP18]] to i32
 ; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP18]], [[CONV61]]
@@ -40,7 +40,7 @@ define void @sdiv_feeding_gep(ptr %dst, i32 %x, i64 %M, i64 %conv6, i64 %N) {
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr double, ptr [[DST]], i64 [[TMP32]]
 ; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr double, ptr [[TMP34]], i32 0
 ; CHECK-NEXT:    [[TMP37:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP38:%.*]] = mul i64 [[TMP37]], 2
+; CHECK-NEXT:    [[TMP38:%.*]] = mul nuw i64 [[TMP37]], 2
 ; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr double, ptr [[TMP34]], i64 [[TMP38]]
 ; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP36]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x double> zeroinitializer, ptr [[TMP39]], align 8
@@ -110,15 +110,15 @@ define void @sdiv_feeding_gep_predicated(ptr %dst, i32 %x, i64 %M, i64 %conv6, i
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[N]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[N]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
@@ -233,15 +233,15 @@ define void @udiv_urem_feeding_gep(i64 %x, ptr %dst, i64 %N) {
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i64 [[TMP0]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = icmp ugt i64 [[TMP0]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
index f957279a7944..c824bee916b0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll
@@ -10,15 +10,15 @@ define void @f1(ptr %A) #0 {
 ; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
index a65d36b51443..847155559c17 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll
@@ -72,16 +72,16 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur
 ; SVE:       for.body.preheader:
 ; SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[SIZE]] to i64
 ; SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SVE:       vector.ph:
 ; SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; SVE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SVE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; SVE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SVE:       vector.body:
 ; SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
index a4dc329c061e..dab14280a6b7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
@@ -13,23 +13,23 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 16
+; DEFAULT-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; DEFAULT:       [[VECTOR_MEMCHECK]]:
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; DEFAULT-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
 ; DEFAULT-NEXT:    [[TMP6:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; DEFAULT-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
 ; DEFAULT-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 16
+; DEFAULT-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 16
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP10]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; DEFAULT-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; DEFAULT-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 [[X]], i64 0
 ; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i32> poison, <vscale x 8 x i32> zeroinitializer
 ; DEFAULT-NEXT:    [[TMP13:%.*]] = trunc <vscale x 8 x i32> [[BROADCAST_SPLAT]] to <vscale x 8 x i16>
@@ -39,7 +39,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[TMP20]], i32 0
 ; DEFAULT-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 8
+; DEFAULT-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
 ; DEFAULT-NEXT:    [[TMP25:%.*]] = getelementptr i8, ptr [[TMP20]], i64 [[TMP24]]
 ; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP22]], align 1
 ; DEFAULT-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP25]], align 1
@@ -58,7 +58,7 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP38:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP40:%.*]] = getelementptr i8, ptr [[TMP38]], i32 0
 ; DEFAULT-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 8
+; DEFAULT-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP41]], 8
 ; DEFAULT-NEXT:    [[TMP43:%.*]] = getelementptr i8, ptr [[TMP38]], i64 [[TMP42]]
 ; DEFAULT-NEXT:    store <vscale x 8 x i8> [[TMP36]], ptr [[TMP40]], align 1
 ; DEFAULT-NEXT:    store <vscale x 8 x i8> [[TMP37]], ptr [[TMP43]], align 1
@@ -98,23 +98,23 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; PRED:       [[VECTOR_MEMCHECK]]:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 16
+; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
 ; PRED-NEXT:    [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
 ; PRED-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; PRED-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP8]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 16
+; PRED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 16
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; PRED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; PRED-NEXT:    [[TMP13:%.*]] = sub i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP14:%.*]] = icmp ugt i64 [[TMP0]], [[TMP12]]
 ; PRED-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i64 [[TMP13]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
index 641564560fc2..6499a1f7e52e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/low_trip_count_predicates.ll
@@ -49,7 +49,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VS1-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
 ; CHECK-VS1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-VS1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; CHECK-VS1:       [[VECTOR_SCEVCHECK]]:
@@ -64,16 +64,16 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK-VS1:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-VS1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 16
+; CHECK-VS1-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 16
 ; CHECK-VS1-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VS1:       [[VECTOR_PH]]:
 ; CHECK-VS1-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 16
+; CHECK-VS1-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 16
 ; CHECK-VS1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
 ; CHECK-VS1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
 ; CHECK-VS1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 16
+; CHECK-VS1-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 16
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
 ; CHECK-VS1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -95,17 +95,17 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS1-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
 ; CHECK-VS1-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
 ; CHECK-VS1-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 8
+; CHECK-VS1-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 8
 ; CHECK-VS1-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
 ; CHECK-VS1-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK-VS1:       [[VEC_EPILOG_PH]]:
 ; CHECK-VS1-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-VS1-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 8
+; CHECK-VS1-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 8
 ; CHECK-VS1-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
 ; CHECK-VS1-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
 ; CHECK-VS1-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS1-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 8
+; CHECK-VS1-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 8
 ; CHECK-VS1-NEXT:    [[TMP39:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS1-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
@@ -155,7 +155,7 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VS2-NEXT:    [[TMP3:%.*]] = sub i64 20, [[TMP2]]
 ; CHECK-VS2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-VS2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]]
 ; CHECK-VS2:       [[VECTOR_SCEVCHECK]]:
@@ -170,16 +170,16 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    br i1 [[TMP13]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK-VS2:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
 ; CHECK-VS2-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-VS2-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 8
 ; CHECK-VS2-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[TMP3]], [[TMP15]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VS2:       [[VECTOR_PH]]:
 ; CHECK-VS2-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 8
+; CHECK-VS2-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 8
 ; CHECK-VS2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP17]]
 ; CHECK-VS2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
 ; CHECK-VS2-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-VS2-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i8> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-VS2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -201,17 +201,17 @@ define void @low_vf_ic_is_better(ptr nocapture noundef %p, i32 %tc, i16 noundef
 ; CHECK-VS2-NEXT:    [[IND_END4:%.*]] = add i64 [[TMP0]], [[N_VEC]]
 ; CHECK-VS2-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 [[TMP3]], [[N_VEC]]
 ; CHECK-VS2-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 4
+; CHECK-VS2-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 4
 ; CHECK-VS2-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP27]]
 ; CHECK-VS2-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK-VS2:       [[VEC_EPILOG_PH]]:
 ; CHECK-VS2-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-VS2-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-VS2-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-VS2-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[TMP3]], [[TMP29]]
 ; CHECK-VS2-NEXT:    [[N_VEC3:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF2]]
 ; CHECK-VS2-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VS2-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 4
+; CHECK-VS2-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 4
 ; CHECK-VS2-NEXT:    [[TMP39:%.*]] = add i64 [[TMP0]], [[N_VEC3]]
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLATINSERT7:%.*]] = insertelement <vscale x 4 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-VS2-NEXT:    [[BROADCAST_SPLAT8:%.*]] = shufflevector <vscale x 4 x i8> [[BROADCAST_SPLATINSERT7]], <vscale x 4 x i8> poison, <vscale x 4 x i32> zeroinitializer
@@ -419,13 +419,13 @@ define void @overflow_indvar_known_false(ptr nocapture noundef %p, i32 noundef %
 ; CHECK-NEXT:    br i1 [[TMP28]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP3]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i64(i64 0, i64 [[TMP1]])
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i8> poison, i8 [[CONV]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i8> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i8> poison, <vscale x 16 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
index 2c0fb797d1d1..6029095bbe7b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll
@@ -15,11 +15,11 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -53,7 +53,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TFCOMMON-NEXT:  [[ENTRY:.*]]:
 ; TFCOMMON-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFCOMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFCOMMON:       [[VECTOR_BODY]]:
@@ -76,9 +76,9 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; TFA_INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -88,7 +88,7 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -96,13 +96,13 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr [[TMP16]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
@@ -138,11 +138,11 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -184,7 +184,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFCOMMON-NEXT:  [[ENTRY:.*]]:
 ; TFCOMMON-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFCOMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFCOMMON:       [[VECTOR_BODY]]:
@@ -210,9 +210,9 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFA_INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -222,7 +222,7 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -236,13 +236,13 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP14]], <vscale x 2 x i64> [[TMP16]], <vscale x 2 x i64> zeroinitializer
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP19]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP17]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP20]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP23:%.*]] = add i64 [[INDEX_NEXT]], [[TMP22]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP23]], i64 1025)
@@ -289,11 +289,11 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -340,7 +340,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFCOMMON-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFCOMMON-NEXT:  [[ENTRY:.*]]:
 ; TFCOMMON-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFCOMMON-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFCOMMON-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFCOMMON-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFCOMMON:       [[VECTOR_BODY]]:
@@ -369,9 +369,9 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFA_INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -381,7 +381,7 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT5:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -401,13 +401,13 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[PREDPHI4:%.*]] = select <vscale x 2 x i1> [[TMP20]], <vscale x 2 x i64> [[TMP22]], <vscale x 2 x i64> [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP23]], i64 [[TMP25]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI]], ptr [[TMP23]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[PREDPHI4]], ptr [[TMP26]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP29:%.*]] = add i64 [[INDEX_NEXT]], [[TMP28]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT5]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP29]], i64 1025)
@@ -457,11 +457,11 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -512,11 +512,11 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFFALLBACK-NEXT:  [[ENTRY:.*]]:
 ; TFFALLBACK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFFALLBACK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFFALLBACK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFFALLBACK-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFFALLBACK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFFALLBACK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFFALLBACK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFFALLBACK:       [[VECTOR_BODY]]:
 ; TFFALLBACK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -586,11 +586,11 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -624,7 +624,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFALWAYS-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFALWAYS-NEXT:  [[ENTRY:.*]]:
 ; TFALWAYS-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFALWAYS-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFALWAYS-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFALWAYS:       [[VECTOR_BODY]]:
@@ -647,7 +647,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFFALLBACK-NEXT:  [[ENTRY:.*]]:
 ; TFFALLBACK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFFALLBACK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFFALLBACK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFFALLBACK:       [[VECTOR_BODY]]:
@@ -670,9 +670,9 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]]) #[[ATTR0]] {
 ; TFA_INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -682,7 +682,7 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK2:%.*]] = phi <vscale x 2 x i1> [ [[ACTIVE_LANE_MASK_ENTRY1]], %[[ENTRY]] ], [ [[ACTIVE_LANE_MASK_NEXT4:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr i64, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr i64, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x i64> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x i64> @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x i64> poison)
@@ -690,13 +690,13 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 {
 ; TFA_INTERLEAVE-NEXT:    [[TMP12:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[WIDE_MASKED_LOAD3]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP11]], ptr [[TMP13]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP12]], ptr [[TMP16]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = add i64 [[INDEX_NEXT]], [[TMP18]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP19]], i64 1025)
@@ -734,11 +734,11 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFNONE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; TFNONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; TFNONE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; TFNONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
 ; TFNONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -783,7 +783,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFALWAYS-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
 ; TFALWAYS-NEXT:  [[ENTRY:.*]]:
 ; TFALWAYS-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFALWAYS-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFALWAYS-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFALWAYS-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFALWAYS-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
 ; TFALWAYS-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
@@ -813,7 +813,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFFALLBACK-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
 ; TFFALLBACK-NEXT:  [[ENTRY:.*]]:
 ; TFFALLBACK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFFALLBACK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFFALLBACK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFFALLBACK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFFALLBACK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
 ; TFFALLBACK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x double> [[BROADCAST_SPLATINSERT]], <vscale x 2 x double> poison, <vscale x 2 x i32> zeroinitializer
@@ -843,9 +843,9 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-SAME: ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], double [[M:%.*]]) #[[ATTR0]] {
 ; TFA_INTERLEAVE-NEXT:  [[ENTRY:.*]]:
 ; TFA_INTERLEAVE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TFA_INTERLEAVE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TFA_INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x double> poison, double [[M]], i64 0
@@ -858,7 +858,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[VEC_PHI:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ]
 ; TFA_INTERLEAVE-NEXT:    [[TMP7:%.*]] = getelementptr double, ptr [[B]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP10:%.*]] = getelementptr double, ptr [[TMP7]], i64 [[TMP9]]
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP7]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]], <vscale x 2 x double> poison)
 ; TFA_INTERLEAVE-NEXT:    [[WIDE_MASKED_LOAD3:%.*]] = call <vscale x 2 x double> @llvm.masked.load.nxv2f64.p0(ptr [[TMP10]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]], <vscale x 2 x double> poison)
@@ -870,7 +870,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP16:%.*]] = call <vscale x 2 x i64> @foo_vector(<vscale x 2 x i64> [[TMP14]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; TFA_INTERLEAVE-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP17]], i64 [[TMP19]]
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP15]], ptr [[TMP17]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK]])
 ; TFA_INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2i64.p0(<vscale x 2 x i64> [[TMP16]], ptr [[TMP20]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
@@ -880,7 +880,7 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub
 ; TFA_INTERLEAVE-NEXT:    [[TMP24]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[TMP22]], <vscale x 2 x double> [[TMP23]])
 ; TFA_INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; TFA_INTERLEAVE-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 2
+; TFA_INTERLEAVE-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 2
 ; TFA_INTERLEAVE-NEXT:    [[TMP27:%.*]] = add i64 [[INDEX_NEXT]], [[TMP26]]
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025)
 ; TFA_INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP27]], i64 1025)
@@ -918,16 +918,16 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFNONE-NEXT:  [[ENTRY:.*]]:
 ; TFNONE-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; TFNONE-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; TFNONE-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; TFNONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; TFNONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TFNONE:       [[VECTOR_PH]]:
 ; TFNONE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TFNONE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TFNONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; TFNONE-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; TFNONE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; TFNONE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; TFNONE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; TFNONE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TFNONE:       [[VECTOR_BODY]]:
 ; TFNONE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -938,7 +938,7 @@ define void @test_widen_exp_v2(ptr noalias %p2, ptr noalias %p, i64 %n) #5 {
 ; TFNONE-NEXT:    [[TMP9:%.*]] = fcmp ogt <vscale x 2 x double> [[TMP8]], zeroinitializer
 ; TFNONE-NEXT:    [[PREDPHI:%.*]] = select <vscale x 2 x i1> [[TMP9]], <vscale x 2 x double> zeroinitializer, <vscale x 2 x double> splat (double 1.000000e+00)
 ; TFNONE-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; TFNONE-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 2
+; TFNONE-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 2
 ; TFNONE-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 1
 ; TFNONE-NEXT:    [[TMP14:%.*]] = extractelement <vscale x 2 x double> [[PREDPHI]], i32 [[TMP13]]
 ; TFNONE-NEXT:    store double [[TMP14]], ptr [[P]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
index e4718dc21635..f0835b818db5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/optsize_minsize.ll
@@ -501,15 +501,15 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; DEFAULT-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; DEFAULT-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; DEFAULT-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; DEFAULT-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; DEFAULT-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -577,15 +577,15 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; OPTSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; OPTSIZE:       [[VECTOR_PH]]:
 ; OPTSIZE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; OPTSIZE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; OPTSIZE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; OPTSIZE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; OPTSIZE-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
 ; OPTSIZE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; OPTSIZE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; OPTSIZE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; OPTSIZE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; OPTSIZE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; OPTSIZE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; OPTSIZE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; OPTSIZE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; OPTSIZE-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; OPTSIZE-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; OPTSIZE-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -653,15 +653,15 @@ define void @sve_tail_predicate_without_minsize(ptr %p, i8 %a, i8 %b, i8 %c, i32
 ; MINSIZE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; MINSIZE:       [[VECTOR_PH]]:
 ; MINSIZE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; MINSIZE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; MINSIZE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; MINSIZE-NEXT:    [[N_RND_UP:%.*]] = add i64 15, [[TMP2]]
 ; MINSIZE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; MINSIZE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; MINSIZE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; MINSIZE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; MINSIZE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; MINSIZE-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; MINSIZE-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; MINSIZE-NEXT:    [[TMP7:%.*]] = sub i64 15, [[TMP6]]
 ; MINSIZE-NEXT:    [[TMP8:%.*]] = icmp ugt i64 15, [[TMP6]]
 ; MINSIZE-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
index 3302103873bd..2cec6ca498e4 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll
@@ -10,16 +10,16 @@ define void @foo() {
 ; CHECK-SAME: () #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP7]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
index a229ca8c6e6d..295fe1b891e0 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-chained.ll
@@ -56,16 +56,16 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -102,16 +102,16 @@ define i32 @chained_partial_reduce_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -221,16 +221,16 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -267,16 +267,16 @@ define i32 @chained_partial_reduce_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -386,16 +386,16 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -432,16 +432,16 @@ define i32 @chained_partial_reduce_sub_add(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -555,16 +555,16 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -601,16 +601,16 @@ define i32 @chained_partial_reduce_sub_sub(ptr %a, ptr %b, ptr %c, i32 %N) #0 {
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -726,16 +726,16 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -774,16 +774,16 @@ define i32 @chained_partial_reduce_add_add_add(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -903,16 +903,16 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-SVE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-SVE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE:       vector.ph:
 ; CHECK-SVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-SVE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-SVE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-SVE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-SVE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE:       vector.body:
 ; CHECK-SVE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -951,16 +951,16 @@ define i32 @chained_partial_reduce_sub_add_sub(ptr %a, ptr %b, ptr %c, i32 %N) #
 ; CHECK-SVE-MAXBW-NEXT:    [[DIV27:%.*]] = lshr i32 [[N]], 1
 ; CHECK-SVE-MAXBW-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext nneg i32 [[DIV27]] to i64
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-SVE-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-SVE-MAXBW:       vector.ph:
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-SVE-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]]
 ; CHECK-SVE-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-SVE-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-SVE-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-SVE-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-SVE-MAXBW:       vector.body:
 ; CHECK-SVE-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
index 66dbcff2c123..400b031917a7 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-epilogue.ll
@@ -9,7 +9,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
@@ -36,18 +36,18 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    br i1 true, label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 0, [[TMP13]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 1024, [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP11]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP15]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[BC_MERGE_RDX]], i32 0
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
index eceff5ede34b..d2a82269e1b8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product-mixed.ll
@@ -10,15 +10,15 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -27,7 +27,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -36,7 +36,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i32 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -60,15 +60,15 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NOI8MM-NEXT:  entry:
 ; CHECK-NOI8MM-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NOI8MM-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-NOI8MM:       vector.ph:
 ; CHECK-NOI8MM-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NOI8MM-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NOI8MM-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -77,7 +77,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-NOI8MM-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -86,7 +86,7 @@ define i32 @sudot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NOI8MM-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -133,15 +133,15 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -150,7 +150,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -159,7 +159,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i32 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
@@ -183,15 +183,15 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-NOI8MM-NEXT:  entry:
 ; CHECK-NOI8MM-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NOI8MM-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-NOI8MM:       vector.ph:
 ; CHECK-NOI8MM-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NOI8MM-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NOI8MM-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NOI8MM-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NOI8MM-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NOI8MM-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-NOI8MM:       vector.body:
 ; CHECK-NOI8MM-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -200,7 +200,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[TMP6]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-NOI8MM-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP7]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP10]], align 1
@@ -209,7 +209,7 @@ define i32 @usdot(ptr %a, ptr %b) #0 {
 ; CHECK-NOI8MM-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NOI8MM-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i32 0
 ; CHECK-NOI8MM-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NOI8MM-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NOI8MM-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-NOI8MM-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x i8>, ptr [[TMP17]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
index b091452e28b4..a44b9bfb3b44 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-dot-product.ll
@@ -11,15 +11,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
@@ -47,15 +47,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT1:%.*]], [[VECTOR_BODY]] ]
@@ -64,7 +64,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP20]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP20]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP21]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -73,7 +73,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX1]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP28]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP28]], i64 [[TMP27]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP9]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -97,15 +97,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -156,15 +156,15 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP18]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP18]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -194,15 +194,15 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP16]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -214,7 +214,7 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP12]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP4]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
@@ -222,7 +222,7 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP1]], i64 [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP6]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
@@ -246,15 +246,15 @@ define i64 @not_dotp_i8_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %b
 ; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -309,15 +309,15 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVE1-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP12]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP14]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP10]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP20]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP10]], 2
@@ -351,15 +351,15 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP11]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP15]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP15]], 2
@@ -375,7 +375,7 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP3:%.*]] = getelementptr i8, ptr [[B]], i64 [[OFFSET_IDX2]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i16>, ptr [[TMP4]], align 2
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i16>, ptr [[TMP30]], align 2
@@ -383,7 +383,7 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = zext <vscale x 2 x i16> [[WIDE_LOAD4]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[NEXT_GEP3]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i16, ptr [[NEXT_GEP3]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i16>, ptr [[TMP8]], align 2
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD6:%.*]] = load <vscale x 2 x i16>, ptr [[TMP21]], align 2
@@ -407,15 +407,15 @@ define i64 @not_dotp_i16_to_i64_has_neon_dotprod(ptr readonly %a, ptr readonly %
 ; CHECK-MAXBW-SAME: ptr readonly [[A:%.*]], ptr readonly [[B:%.*]]) #[[ATTR1]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 2
@@ -833,17 +833,17 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -866,11 +866,11 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP20]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = sub i32 [[TMP21]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP23:%.*]] = extractelement <vscale x 8 x i32> [[TMP18]], i32 [[TMP22]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP25:%.*]] = mul i32 [[TMP24]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP25:%.*]] = mul nuw i32 [[TMP24]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP25]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP16]], i32 [[TMP26]]
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -881,22 +881,22 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP10]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP11]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
@@ -905,7 +905,7 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP17]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr i8, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP18]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x i8>, ptr [[TMP21]], align 1
@@ -920,11 +920,11 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = mul nuw i32 [[TMP29]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP32:%.*]] = extractelement <vscale x 8 x i32> [[TMP27]], i32 [[TMP31]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = mul i32 [[TMP33]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = mul nuw i32 [[TMP33]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP34]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP25]], i32 [[TMP35]]
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -935,17 +935,17 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 8
+; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 8 x i32> poison, i32 0, i32 [[TMP8]]
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -968,11 +968,11 @@ define i32 @not_dotp_not_loop_carried(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 8
+; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP20]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP22:%.*]] = sub i32 [[TMP21]], 1
 ; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = extractelement <vscale x 8 x i32> [[TMP27]], i32 [[TMP22]]
 ; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP24]], 8
+; CHECK-MAXBW-NEXT:    [[TMP30:%.*]] = mul nuw i32 [[TMP24]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
 ; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP25]], i32 [[TMP31]]
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1006,15 +1006,15 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1033,11 +1033,11 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul i32 [[TMP19]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = mul nuw i32 [[TMP19]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP21:%.*]] = sub i32 [[TMP20]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP22:%.*]] = extractelement <vscale x 8 x i32> [[TMP17]], i32 [[TMP21]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP24:%.*]] = mul i32 [[TMP23]], 8
+; CHECK-INTERLEAVE1-NEXT:    [[TMP24:%.*]] = mul nuw i32 [[TMP23]], 8
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP24]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP17]], i32 [[TMP25]]
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1048,27 +1048,27 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP14]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD]] to <vscale x 8 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = getelementptr i8, ptr [[TMP16]], i64 [[TMP19]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x i8>, ptr [[TMP20]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = zext <vscale x 8 x i8> [[WIDE_LOAD1]] to <vscale x 8 x i32>
@@ -1079,11 +1079,11 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[TMP24]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-INTERLEAVED:       middle.block:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP23]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP31:%.*]] = mul nuw i32 [[TMP23]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP31]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i32 [[TMP25]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = mul nuw i32 [[TMP27]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP28]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i32 [[TMP29]]
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1094,15 +1094,15 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1121,11 +1121,11 @@ define i32 @not_dotp_not_phi(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 8
+; CHECK-MAXBW-NEXT:    [[TMP27:%.*]] = mul nuw i32 [[TMP26]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP27]], 1
 ; CHECK-MAXBW-NEXT:    [[TMP29:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i32 [[TMP28]]
 ; CHECK-MAXBW-NEXT:    [[TMP23:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = mul i32 [[TMP23]], 8
+; CHECK-MAXBW-NEXT:    [[TMP24:%.*]] = mul nuw i32 [[TMP23]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP25:%.*]] = sub i32 [[TMP24]], 1
 ; CHECK-MAXBW-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 8 x i32> [[TMP21]], i32 [[TMP25]]
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1159,16 +1159,16 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP13]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP15]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP18]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1235,16 +1235,16 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP13]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP13]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP16]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP34:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP34]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP34]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1269,7 +1269,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 [[TMP9]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP56]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP56]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i64 [[TMP20]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP12]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 4 x i8>, ptr [[TMP21]], align 1
@@ -1277,7 +1277,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP23:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD8]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP72:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i64 [[TMP26]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD9:%.*]] = load <vscale x 4 x i8>, ptr [[TMP14]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD10:%.*]] = load <vscale x 4 x i8>, ptr [[TMP72]], align 1
@@ -1289,7 +1289,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP33]] = add <vscale x 4 x i32> [[TMP31]], [[VEC_PHI7]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP36:%.*]] = mul nuw i64 [[TMP35]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i8, ptr [[TMP4]], i64 [[TMP36]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD11:%.*]] = load <vscale x 4 x i8>, ptr [[TMP17]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD12:%.*]] = load <vscale x 4 x i8>, ptr [[TMP37]], align 1
@@ -1297,7 +1297,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP39:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD12]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP41]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP43:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i64 [[TMP42]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 4 x i8>, ptr [[TMP19]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD14:%.*]] = load <vscale x 4 x i8>, ptr [[TMP43]], align 1
@@ -1309,7 +1309,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP49]] = add <vscale x 4 x i32> [[TMP47]], [[VEC_PHI5]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP51]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP53:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i64 [[TMP52]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD15:%.*]] = load <vscale x 4 x i8>, ptr [[TMP22]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD16:%.*]] = load <vscale x 4 x i8>, ptr [[TMP53]], align 1
@@ -1317,7 +1317,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP55:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD16]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP57:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = mul i64 [[TMP57]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP58:%.*]] = mul nuw i64 [[TMP57]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP59:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[TMP58]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD17:%.*]] = load <vscale x 4 x i8>, ptr [[TMP24]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD18:%.*]] = load <vscale x 4 x i8>, ptr [[TMP59]], align 1
@@ -1329,7 +1329,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP65]] = add <vscale x 4 x i32> [[TMP63]], [[VEC_PHI3]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP67:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = mul i64 [[TMP67]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP68:%.*]] = mul nuw i64 [[TMP67]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP69:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 [[TMP68]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD19:%.*]] = load <vscale x 4 x i8>, ptr [[TMP27]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD20:%.*]] = load <vscale x 4 x i8>, ptr [[TMP69]], align 1
@@ -1337,7 +1337,7 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP71:%.*]] = sext <vscale x 4 x i8> [[WIDE_LOAD20]] to <vscale x 4 x i32>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP73:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = mul i64 [[TMP73]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP74:%.*]] = mul nuw i64 [[TMP73]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP75:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i64 [[TMP74]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD21:%.*]] = load <vscale x 4 x i8>, ptr [[TMP29]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD22:%.*]] = load <vscale x 4 x i8>, ptr [[TMP75]], align 1
@@ -1367,16 +1367,16 @@ define i32 @dotp_unrolled(i32 %num_out, i64 %num_in, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: i32 [[NUM_OUT:%.*]], i64 [[NUM_IN:%.*]], ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[NUM_IN]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[NUM_IN]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[NUM_IN]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1501,15 +1501,15 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1546,15 +1546,15 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1591,15 +1591,15 @@ define i32 @dotp_predicated(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-MAXBW-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1657,15 +1657,15 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP10]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP11]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1686,7 +1686,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1:       middle.block:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[TMP14]])
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 4 x i32> [[TMP12]], i32 [[TMP19]]
 ; CHECK-INTERLEAVE1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1697,15 +1697,15 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP7]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP14]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP14]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1714,7 +1714,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = getelementptr i8, ptr [[TMP3]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP4]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -1723,7 +1723,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP8]], i64 [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP9]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -1740,7 +1740,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[BIN_RDX:%.*]] = add <vscale x 4 x i32> [[TMP24]], [[TMP23]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
 ; CHECK-INTERLEAVED-NEXT:    [[TMP27:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = mul i32 [[TMP27]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP28:%.*]] = mul nuw i32 [[TMP27]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP29:%.*]] = sub i32 [[TMP28]], 1
 ; CHECK-INTERLEAVED-NEXT:    [[TMP30:%.*]] = extractelement <vscale x 4 x i32> [[TMP20]], i32 [[TMP29]]
 ; CHECK-INTERLEAVED-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1751,15 +1751,15 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1780,7 +1780,7 @@ define i32 @not_dotp_extend_user(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW:       middle.block:
 ; CHECK-MAXBW-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32(<vscale x 8 x i32> [[TMP24]])
 ; CHECK-MAXBW-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 8
+; CHECK-MAXBW-NEXT:    [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
 ; CHECK-MAXBW-NEXT:    [[TMP21:%.*]] = extractelement <vscale x 8 x i32> [[TMP20]], i32 [[TMP19]]
 ; CHECK-MAXBW-NEXT:    [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]]
@@ -1815,16 +1815,16 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 16, [[TMP1]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 16, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 16, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1853,16 +1853,16 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 16, [[TMP1]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 16, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 16, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1871,7 +1871,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP8]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 2 x i8>, ptr [[TMP11]], align 1
@@ -1881,7 +1881,7 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = getelementptr inbounds nuw i8, ptr [[B]], i64 [[TMP14]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP15]], i64 [[TMP18]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP16]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
@@ -1905,16 +1905,16 @@ define i64 @dotp_cost_disagreement(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 16, [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 16, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 16, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2218,16 +2218,16 @@ define i64 @not_dotp_ext_outside_plan(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW:       for.ph:
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i16 [[B]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[EXT_B]], i64 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -2356,16 +2356,16 @@ define i64 @not_dotp_ext_outside_plan2(ptr %a, i16 %b, i64 %n) #0 {
 ; CHECK-MAXBW:       for.ph:
 ; CHECK-MAXBW-NEXT:    [[EXT_B:%.*]] = zext i16 [[B]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[EXT_B]], i64 0
 ; CHECK-MAXBW-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -2421,16 +2421,16 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVE1:       for.body.preheader:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-INTERLEAVE1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-INTERLEAVE1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2466,16 +2466,16 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED:       for.body.preheader:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-INTERLEAVED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2489,7 +2489,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i8>, ptr [[TMP13]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i8>, ptr [[TMP16]], align 1
@@ -2497,7 +2497,7 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = zext <vscale x 2 x i8> [[WIDE_LOAD3]] to <vscale x 2 x i64>
 ; CHECK-INTERLEAVED-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 2
+; CHECK-INTERLEAVED-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 2
 ; CHECK-INTERLEAVED-NEXT:    [[TMP22:%.*]] = getelementptr i8, ptr [[NEXT_GEP2]], i64 [[TMP21]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 2 x i8>, ptr [[TMP19]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 2 x i8>, ptr [[TMP22]], align 1
@@ -2525,16 +2525,16 @@ define dso_local i32 @not_dotp_vscale1(ptr %a, ptr %b, i32 %n, i64 %cost) #0 {
 ; CHECK-MAXBW:       for.body.preheader:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-MAXBW-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-MAXBW-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-MAXBW-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-MAXBW-NEXT:    [[TMP7:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-MAXBW-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[A]], i64 [[N_VEC]]
 ; CHECK-MAXBW-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[B]], i64 [[N_VEC]]
@@ -2647,15 +2647,15 @@ define i32 @zext_add_reduc_i8_i32(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2748,15 +2748,15 @@ define i64 @zext_add_reduc_i8_i64(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2849,15 +2849,15 @@ define i64 @zext_add_reduc_i16_i64(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2950,15 +2950,15 @@ define i64 @zext_add_reduc_i8_i64_has_neon_dotprod(ptr %a) #1 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR1]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -3051,15 +3051,15 @@ define i32 @sext_add_reduc_i8_i32(ptr %a) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
index 3514404d3b2d..a471c004a8de 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-sub.ll
@@ -11,15 +11,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVE1-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVE1-NEXT:  entry:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVE1:       vector.ph:
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-INTERLEAVE1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVE1-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVE1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-INTERLEAVE1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-INTERLEAVE1-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVE1:       vector.body:
 ; CHECK-INTERLEAVE1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -48,15 +48,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-INTERLEAVED-NEXT:  entry:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-INTERLEAVED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-INTERLEAVED:       vector.ph:
 ; CHECK-INTERLEAVED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-INTERLEAVED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-INTERLEAVED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-INTERLEAVED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-INTERLEAVED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-INTERLEAVED:       vector.body:
 ; CHECK-INTERLEAVED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -65,7 +65,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP7:%.*]] = getelementptr i8, ptr [[A]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP8:%.*]] = getelementptr i8, ptr [[TMP7]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP11:%.*]] = getelementptr i8, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i8>, ptr [[TMP8]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i8>, ptr [[TMP11]], align 1
@@ -74,7 +74,7 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-INTERLEAVED-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[B]], i64 [[INDEX]]
 ; CHECK-INTERLEAVED-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[TMP14]], i32 0
 ; CHECK-INTERLEAVED-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-INTERLEAVED-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-INTERLEAVED-NEXT:    [[TMP18:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP17]]
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i8>, ptr [[TMP15]], align 1
 ; CHECK-INTERLEAVED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x i8>, ptr [[TMP18]], align 1
@@ -100,15 +100,15 @@ define i32 @dotp(ptr %a, ptr %b) #0 {
 ; CHECK-MAXBW-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-MAXBW-NEXT:  entry:
 ; CHECK-MAXBW-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-MAXBW-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-MAXBW-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; CHECK-MAXBW:       vector.ph:
 ; CHECK-MAXBW-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-MAXBW-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-MAXBW-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-MAXBW-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-MAXBW-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-MAXBW-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-MAXBW-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK-MAXBW:       vector.body:
 ; CHECK-MAXBW-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
index b92d8406f614..6d5bbde36642 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll
@@ -9,15 +9,15 @@ define void @test_invar_gep(ptr %dst) #0 {
 ; CHECK-LABEL: @test_invar_gep(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -32,7 +32,7 @@ define void @test_invar_gep(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[TMP13:%.*]] = add i64 [[INDEX]], 3
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP16]], 1
 ; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <vscale x 4 x i64> [[TMP9]], i32 [[TMP17]]
 ; CHECK-NEXT:    store i64 [[TMP18]], ptr [[TMP14:%.*]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
index c2fe37ad214c..492ab5632473 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/reduction-recurrence-costs-sve.ll
@@ -51,16 +51,16 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:  [[ENTRY:.*]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[Y]], 1
 ; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VSCALEFORTUNING2:       [[VECTOR_PH]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
@@ -76,11 +76,11 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[TMP15:%.*]] = zext <vscale x 4 x i32> [[TMP14]] to <vscale x 4 x i64>
 ; VSCALEFORTUNING2-NEXT:    [[DOTSPLAT:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP15]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP18:%.*]] = call i32 @llvm.vscale.i32()
-; VSCALEFORTUNING2-NEXT:    [[TMP19:%.*]] = mul i32 [[TMP18]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP19:%.*]] = mul nuw i32 [[TMP18]], 4
 ; VSCALEFORTUNING2-NEXT:    [[TMP20:%.*]] = sub i32 [[TMP19]], 1
 ; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP20]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
-; VSCALEFORTUNING2-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP22:%.*]] = mul nuw i32 [[TMP21]], 4
 ; VSCALEFORTUNING2-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP22]], 1
 ; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP23]]
 ; VSCALEFORTUNING2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -128,7 +128,7 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; VSCALEFORTUNING2-NEXT:    [[BIN_RDX:%.*]] = or <vscale x 4 x i32> [[TMP48]], [[TMP47]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP50:%.*]] = call i32 @llvm.vector.reduce.or.nxv4i32(<vscale x 4 x i32> [[BIN_RDX]])
 ; VSCALEFORTUNING2-NEXT:    [[TMP51:%.*]] = call i32 @llvm.vscale.i32()
-; VSCALEFORTUNING2-NEXT:    [[TMP52:%.*]] = mul i32 [[TMP51]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP52:%.*]] = mul nuw i32 [[TMP51]], 4
 ; VSCALEFORTUNING2-NEXT:    [[TMP53:%.*]] = sub i32 [[TMP52]], 1
 ; VSCALEFORTUNING2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP26]], i32 [[TMP53]]
 ; VSCALEFORTUNING2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -181,19 +181,19 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; PRED-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP3]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; PRED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[Z]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; PRED-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP0]], [[TMP7]]
 ; PRED-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[TMP0]], [[TMP7]]
 ; PRED-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
@@ -209,11 +209,11 @@ define i32 @chained_recurrences(i32 %x, i64 %y, ptr %src.1, i32 %z, ptr %src.2)
 ; PRED-NEXT:    [[TMP19:%.*]] = zext <vscale x 4 x i32> [[TMP18]] to <vscale x 4 x i64>
 ; PRED-NEXT:    [[DOTSPLAT:%.*]] = getelementptr i32, ptr [[SRC_2]], <vscale x 4 x i64> [[TMP19]]
 ; PRED-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 4
+; PRED-NEXT:    [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 4
 ; PRED-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
 ; PRED-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP24]]
 ; PRED-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
-; PRED-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 4
+; PRED-NEXT:    [[TMP26:%.*]] = mul nuw i32 [[TMP25]], 4
 ; PRED-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP26]], 1
 ; PRED-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 0, i32 [[TMP27]]
 ; PRED-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -335,16 +335,16 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:  [[ENTRY:.*]]:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; DEFAULT-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; DEFAULT-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; DEFAULT:       [[VECTOR_PH]]:
 ; DEFAULT-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; DEFAULT-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; DEFAULT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; DEFAULT-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; DEFAULT-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
 ; DEFAULT-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
 ; DEFAULT-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -355,7 +355,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; DEFAULT-NEXT:    [[TMP13:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP15:%.*]] = getelementptr i16, ptr [[TMP13]], i32 0
 ; DEFAULT-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; DEFAULT-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; DEFAULT-NEXT:    [[TMP18:%.*]] = getelementptr i16, ptr [[TMP13]], i64 [[TMP17]]
 ; DEFAULT-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP15]], align 2
 ; DEFAULT-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP18]], align 2
@@ -394,16 +394,16 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:  [[ENTRY:.*]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP0:%.*]] = add i64 [[N]], 1
 ; VSCALEFORTUNING2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; VSCALEFORTUNING2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; VSCALEFORTUNING2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; VSCALEFORTUNING2:       [[VECTOR_PH]]:
 ; VSCALEFORTUNING2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; VSCALEFORTUNING2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; VSCALEFORTUNING2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; VSCALEFORTUNING2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[X]], i64 0
 ; VSCALEFORTUNING2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i16> poison, <vscale x 4 x i32> zeroinitializer
 ; VSCALEFORTUNING2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -414,7 +414,7 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; VSCALEFORTUNING2-NEXT:    [[TMP8:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[INDEX]]
 ; VSCALEFORTUNING2-NEXT:    [[TMP9:%.*]] = getelementptr i16, ptr [[TMP8]], i32 0
 ; VSCALEFORTUNING2-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; VSCALEFORTUNING2-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; VSCALEFORTUNING2-NEXT:    [[TMP12:%.*]] = getelementptr i16, ptr [[TMP8]], i64 [[TMP11]]
 ; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP9]], align 2
 ; VSCALEFORTUNING2-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x i16>, ptr [[TMP12]], align 2
@@ -455,15 +455,15 @@ define i16 @reduce_udiv(ptr %src, i16 %x, i64 %N) #0 {
 ; PRED-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; PRED:       [[VECTOR_PH]]:
 ; PRED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; PRED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; PRED-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP2]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP5]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; PRED-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; PRED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; PRED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; PRED-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP0]], [[TMP9]]
 ; PRED-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP0]], [[TMP9]]
 ; PRED-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
index ba58c32bb3f5..0f2eae10f4ac 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll
@@ -15,16 +15,16 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
 ; CHECK-NEXT:    [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N:%.*]], i32 [[TMP0]])
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i32 [[SMAX]], [[IDX]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i32 [[IDX]], [[N_VEC]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 2 x i32> @llvm.stepvector.nxv2i32()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[IDX]], i64 0
@@ -52,7 +52,7 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0
 ; CHECK-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], 2
+; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i32 [[TMP21]], 2
 ; CHECK-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP22]], 1
 ; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <vscale x 2 x ptr> [[TMP15]], i32 [[TMP23]]
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
index 68276c2c8af7..1b489dd25b92 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll
@@ -6,16 +6,16 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture
 ; CHECK-LABEL: @cond_fadd(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -92,16 +92,16 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) {
 ; CHECK-LABEL: @cond_cmp_sel(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
index 82f64e988e6d..d73cdc1228fe 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll
@@ -34,16 +34,16 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -80,16 +80,16 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -127,15 +127,15 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -217,16 +217,16 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -237,13 +237,13 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -284,16 +284,16 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -301,13 +301,13 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -346,26 +346,26 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
@@ -382,13 +382,13 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
@@ -404,13 +404,13 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP34]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP32]], <vscale x 8 x float> [[TMP33]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul nuw i64 [[TMP35]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = add i64 [[INDEX]], [[TMP36]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP38:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = mul i64 [[TMP38]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP39:%.*]] = mul nuw i64 [[TMP38]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP40:%.*]] = add i64 [[INDEX]], [[TMP39]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP41:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = mul i64 [[TMP41]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP42:%.*]] = mul nuw i64 [[TMP41]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP43:%.*]] = add i64 [[INDEX]], [[TMP42]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT12]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP37]], i64 [[TMP9]])
@@ -498,16 +498,16 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A2]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[A1]], i32 0
@@ -568,16 +568,16 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
@@ -636,15 +636,15 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP5]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
@@ -768,16 +768,16 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-UNORDERED-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK-UNORDERED:       for.body.preheader:
 ; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -829,16 +829,16 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-NEXT:    br i1 [[CMP1]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_END:%.*]]
 ; CHECK-ORDERED:       for.body.preheader:
 ; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -891,15 +891,15 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = sub i64 [[TMP2]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = sub i64 [[N]], [[TMP7]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = icmp ugt i64 [[N]], [[TMP7]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i64 [[TMP8]], i64 0
@@ -1006,16 +1006,16 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-UNORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1065,16 +1065,16 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-ORDERED-SAME: (ptr noalias readonly captures(none) [[A:%.*]], ptr noalias readonly captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1125,15 +1125,15 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -1244,16 +1244,16 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b,
 ; CHECK-UNORDERED-SAME: (ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1383,16 +1383,16 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1403,13 +1403,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -1418,13 +1418,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
 ; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
 ; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
 ; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 24
 ; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
@@ -1467,16 +1467,16 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1484,13 +1484,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -1499,13 +1499,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
 ; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
 ; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
 ; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 24
 ; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
@@ -1550,26 +1550,26 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
@@ -1586,13 +1586,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
@@ -1601,13 +1601,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP30]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul nuw i64 [[TMP32]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP33]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul nuw i64 [[TMP35]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP36]]
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP28]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP31]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
@@ -1627,13 +1627,13 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP49]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP47]], <vscale x 8 x float> [[TMP48]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul i64 [[TMP50]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul nuw i64 [[TMP50]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = add i64 [[INDEX]], [[TMP51]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul i64 [[TMP53]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul nuw i64 [[TMP53]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = add i64 [[INDEX]], [[TMP54]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul nuw i64 [[TMP56]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX]], [[TMP57]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP52]], i64 [[TMP9]])
@@ -1711,16 +1711,16 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-UNORDERED-NEXT:  entry:
 ; CHECK-UNORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-UNORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-UNORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-UNORDERED:       vector.ph:
 ; CHECK-UNORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-UNORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-UNORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-UNORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-UNORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-UNORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-UNORDERED:       vector.body:
 ; CHECK-UNORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1731,13 +1731,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-UNORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-UNORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-UNORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-UNORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-UNORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-UNORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -1746,13 +1746,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-UNORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-UNORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; CHECK-UNORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-UNORDERED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
 ; CHECK-UNORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-UNORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-UNORDERED-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
 ; CHECK-UNORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
 ; CHECK-UNORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-UNORDERED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 24
 ; CHECK-UNORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD7:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
 ; CHECK-UNORDERED-NEXT:    [[WIDE_LOAD8:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
@@ -1795,16 +1795,16 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-SAME: (ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-ORDERED-NEXT:  entry:
 ; CHECK-ORDERED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-ORDERED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED:       vector.ph:
 ; CHECK-ORDERED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-ORDERED-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-ORDERED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-ORDERED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-ORDERED-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-ORDERED-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-ORDERED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-ORDERED:       vector.body:
 ; CHECK-ORDERED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1812,13 +1812,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-NEXT:    [[TMP7:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i32 0
 ; CHECK-ORDERED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-ORDERED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-ORDERED-NEXT:    [[TMP10:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP9]]
 ; CHECK-ORDERED-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 16
+; CHECK-ORDERED-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 16
 ; CHECK-ORDERED-NEXT:    [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP12]]
 ; CHECK-ORDERED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[TMP6]], i64 [[TMP15]]
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x float>, ptr [[TMP7]], align 4
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 8 x float>, ptr [[TMP10]], align 4
@@ -1827,13 +1827,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-ORDERED-NEXT:    [[TMP18:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i32 0
 ; CHECK-ORDERED-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 8
+; CHECK-ORDERED-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 8
 ; CHECK-ORDERED-NEXT:    [[TMP21:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP20]]
 ; CHECK-ORDERED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; CHECK-ORDERED-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
 ; CHECK-ORDERED-NEXT:    [[TMP24:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP23]]
 ; CHECK-ORDERED-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 24
+; CHECK-ORDERED-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 24
 ; CHECK-ORDERED-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[TMP26]]
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 8 x float>, ptr [[TMP18]], align 4
 ; CHECK-ORDERED-NEXT:    [[WIDE_LOAD5:%.*]] = load <vscale x 8 x float>, ptr [[TMP21]], align 4
@@ -1878,26 +1878,26 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-ORDERED-TF:       vector.ph:
 ; CHECK-ORDERED-TF-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-ORDERED-TF-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP2]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-ORDERED-TF-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 32
+; CHECK-ORDERED-TF-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 32
 ; CHECK-ORDERED-TF-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP11]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP13]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP15]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 0, i64 [[N]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[N]])
@@ -1914,13 +1914,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP20:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP19]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP23:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP22]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP26:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[TMP25]]
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP17]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD9:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP20]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
@@ -1929,13 +1929,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP27:%.*]] = getelementptr inbounds float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP28:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i32 0
 ; CHECK-ORDERED-TF-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP31:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP30]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP33:%.*]] = mul nuw i64 [[TMP32]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP34:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP33]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP35:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul i64 [[TMP35]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP36:%.*]] = mul nuw i64 [[TMP35]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP37:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[TMP36]]
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD12:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP28]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK]], <vscale x 8 x float> poison)
 ; CHECK-ORDERED-TF-NEXT:    [[WIDE_MASKED_LOAD13:%.*]] = call <vscale x 8 x float> @llvm.masked.load.nxv8f32.p0(ptr [[TMP31]], i32 4, <vscale x 8 x i1> [[ACTIVE_LANE_MASK6]], <vscale x 8 x float> poison)
@@ -1955,13 +1955,13 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 {
 ; CHECK-ORDERED-TF-NEXT:    [[TMP49]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP47]], <vscale x 8 x float> [[TMP48]])
 ; CHECK-ORDERED-TF-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP50:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul i64 [[TMP50]], 8
+; CHECK-ORDERED-TF-NEXT:    [[TMP51:%.*]] = mul nuw i64 [[TMP50]], 8
 ; CHECK-ORDERED-TF-NEXT:    [[TMP52:%.*]] = add i64 [[INDEX]], [[TMP51]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP53:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul i64 [[TMP53]], 16
+; CHECK-ORDERED-TF-NEXT:    [[TMP54:%.*]] = mul nuw i64 [[TMP53]], 16
 ; CHECK-ORDERED-TF-NEXT:    [[TMP55:%.*]] = add i64 [[INDEX]], [[TMP54]]
 ; CHECK-ORDERED-TF-NEXT:    [[TMP56:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul i64 [[TMP56]], 24
+; CHECK-ORDERED-TF-NEXT:    [[TMP57:%.*]] = mul nuw i64 [[TMP56]], 24
 ; CHECK-ORDERED-TF-NEXT:    [[TMP58:%.*]] = add i64 [[INDEX]], [[TMP57]]
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]])
 ; CHECK-ORDERED-TF-NEXT:    [[ACTIVE_LANE_MASK_NEXT16]] = call <vscale x 8 x i1> @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[TMP52]], i64 [[TMP9]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
index d02d03b4b437..5925b7014b12 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll
@@ -14,16 +14,16 @@ define i64 @same_exit_block_pre_inc_use1() #1 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 64, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = add i64 3, [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -260,16 +260,16 @@ define i64 @loop_contains_safe_div() #1 {
 ; CHECK-NEXT:    call void @init_mem(ptr [[P1]], i64 1024)
 ; CHECK-NEXT:    call void @init_mem(ptr [[P2]], i64 1024)
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP12]])
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[INDEX1:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
index 15c5258b57cc..77b768e45e89 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/store-costs-sve.ll
@@ -10,28 +10,28 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT-SAME: ptr [[DST:%.*]]) #[[ATTR0:[0-9]+]] {
 ; DEFAULT-NEXT:  iter.check:
 ; DEFAULT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; DEFAULT-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP1]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; DEFAULT:       vector.main.loop.iter.check:
 ; DEFAULT-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; DEFAULT-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; DEFAULT-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 101, [[TMP3]]
 ; DEFAULT-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DEFAULT:       vector.ph:
 ; DEFAULT-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; DEFAULT-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; DEFAULT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 101, [[TMP5]]
 ; DEFAULT-NEXT:    [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
 ; DEFAULT-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 32
+; DEFAULT-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; DEFAULT-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; DEFAULT:       vector.body:
 ; DEFAULT-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; DEFAULT-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; DEFAULT-NEXT:    [[TMP10:%.*]] = getelementptr i8, ptr [[TMP9]], i32 0
 ; DEFAULT-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 16
+; DEFAULT-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 16
 ; DEFAULT-NEXT:    [[TMP24:%.*]] = getelementptr i8, ptr [[TMP9]], i64 [[TMP23]]
 ; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP10]], align 1
 ; DEFAULT-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP24]], align 1
@@ -44,17 +44,17 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; DEFAULT:       vec.epilog.iter.check:
 ; DEFAULT-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 101, [[N_VEC]]
 ; DEFAULT-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
+; DEFAULT-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
 ; DEFAULT-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP13]]
 ; DEFAULT-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; DEFAULT:       vec.epilog.ph:
 ; DEFAULT-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; DEFAULT-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 8
+; DEFAULT-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 8
 ; DEFAULT-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 101, [[TMP15]]
 ; DEFAULT-NEXT:    [[N_VEC3:%.*]] = sub i64 101, [[N_MOD_VF2]]
 ; DEFAULT-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; DEFAULT-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 8
+; DEFAULT-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 8
 ; DEFAULT-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; DEFAULT:       vec.epilog.vector.body:
 ; DEFAULT-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
@@ -86,15 +86,15 @@ define void @cost_store_i8(ptr %dst) #0 {
 ; PRED-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; PRED:       vector.ph:
 ; PRED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; PRED-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 101, [[TMP4]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; PRED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; PRED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 16
+; PRED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 16
 ; PRED-NEXT:    [[TMP9:%.*]] = sub i64 101, [[TMP8]]
 ; PRED-NEXT:    [[TMP10:%.*]] = icmp ugt i64 101, [[TMP8]]
 ; PRED-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 0
@@ -231,13 +231,13 @@ define void @trunc_store(ptr %dst, ptr %src, i16 %x) #1 {
 ; PRED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; PRED:       vector.ph:
 ; PRED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP10]], 2
+; PRED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP10]], 2
 ; PRED-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; PRED-NEXT:    [[N_RND_UP:%.*]] = add i64 1000, [[TMP2]]
 ; PRED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; PRED-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; PRED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; PRED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; PRED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; PRED-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i16> poison, i16 [[X]], i64 0
 ; PRED-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i16> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i16> poison, <vscale x 2 x i32> zeroinitializer
 ; PRED-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1000)
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
index 36e9befb606d..0fddadd4e3ac 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll
@@ -11,16 +11,16 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -29,7 +29,7 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) {
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP12]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP15]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-no-remaining-iterations.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-no-remaining-iterations.ll
index f8551d774de4..d85bc484af0b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-no-remaining-iterations.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-no-remaining-iterations.ll
@@ -8,7 +8,7 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ITER_CHECK:.*]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 16, [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
 ; CHECK:       [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
@@ -40,20 +40,20 @@ define i64 @main_vector_loop_fixed_with_no_remaining_iterations(ptr %src, ptr no
 ; CHECK-NEXT:    br label %[[VEC_EPILOG_ITER_CHECK:.*]]
 ; CHECK:       [[VEC_EPILOG_ITER_CHECK]]:
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ule i64 16, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]]
 ; CHECK:       [[VEC_EPILOG_PH]]:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ 0, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP18]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP31:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP31]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP31]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 16, [[TMP16]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP36:%.*]] = select i1 [[TMP32]], i64 [[TMP16]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 16, [[TMP36]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 2
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <vscale x 2 x i64> zeroinitializer, i64 [[BC_MERGE_RDX]], i32 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
index e5633462973a..de8fcb0aff7e 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll
@@ -11,16 +11,16 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -29,7 +29,7 @@ define i64 @int_reduction_add(ptr %a, i64 %N) {
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 2
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 2
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP12]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 2 x i64>, ptr [[TMP15]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
index 7e49c69266d8..83f2b2a9080a 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll
@@ -11,16 +11,16 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -28,7 +28,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) {
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
index 5d0e6f72309c..520937454ce5 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll
@@ -22,28 +22,28 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-LABEL: @main_vf_vscale_x_16(
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 1024, [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 16
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 16
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP16]], align 1
 ; CHECK-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP19]], align 1
@@ -56,17 +56,17 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 1024, [[N_VEC]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 8
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP22]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 1024, [[TMP24]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 1024, [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
+; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 8
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ]
@@ -89,23 +89,23 @@ define void @main_vf_vscale_x_16(ptr %A) #0 {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
 ; CHECK-VF8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 16
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 16
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP14]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> splat (i8 1), ptr [[TMP17]], align 1
@@ -173,23 +173,23 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP14]], align 1
 ; CHECK-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP17]], align 1
@@ -226,23 +226,23 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
 ; CHECK-VF8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 2
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP14]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 2 x i64> splat (i64 1), ptr [[TMP17]], align 1
@@ -295,28 +295,28 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-LABEL: @test_pr57912_pointer_induction(
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 10000, [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 32
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 32
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP14]], i32 0
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP17]], 16
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP17]], 16
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i8, ptr [[TMP14]], i64 [[TMP18]]
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP16]], align 1
 ; CHECK-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP19]], align 1
@@ -330,17 +330,17 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-NEXT:    [[IND_END4:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC]]
 ; CHECK-NEXT:    [[N_VEC_REMAINING:%.*]] = sub i64 10000, [[N_VEC]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 8
+; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 8
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], [[TMP22]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
 ; CHECK-NEXT:    [[TMP23:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP24:%.*]] = mul i64 [[TMP23]], 8
+; CHECK-NEXT:    [[TMP24:%.*]] = mul nuw i64 [[TMP23]], 8
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 10000, [[TMP24]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 10000, [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 8
+; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC3]]
 ; CHECK-NEXT:    br label [[VEC_EPILOG_VECTOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
@@ -365,23 +365,23 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 {
 ; CHECK-VF8-NEXT:    br i1 false, label [[VEC_EPILOG_SCALAR_PH:%.*]], label [[VECTOR_MAIN_LOOP_ITER_CHECK:%.*]]
 ; CHECK-VF8:       vector.main.loop.iter.check:
 ; CHECK-VF8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 32
+; CHECK-VF8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 32
 ; CHECK-VF8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 10000, [[TMP1]]
 ; CHECK-VF8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-VF8:       vector.ph:
 ; CHECK-VF8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 32
+; CHECK-VF8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 32
 ; CHECK-VF8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]]
 ; CHECK-VF8-NEXT:    [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]]
 ; CHECK-VF8-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 32
+; CHECK-VF8-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 32
 ; CHECK-VF8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK-VF8:       vector.body:
 ; CHECK-VF8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-VF8-NEXT:    [[TMP12:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[INDEX]]
 ; CHECK-VF8-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP12]], i32 0
 ; CHECK-VF8-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 16
+; CHECK-VF8-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 16
 ; CHECK-VF8-NEXT:    [[TMP17:%.*]] = getelementptr i8, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP14]], align 1
 ; CHECK-VF8-NEXT:    store <vscale x 16 x i8> zeroinitializer, ptr [[TMP17]], align 1
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-extract-last-veclane.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-extract-last-veclane.ll
index a01e3e169298..7e24eb03c290 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-extract-last-veclane.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-extract-last-veclane.ll
@@ -9,7 +9,7 @@ define void @inv_store_last_lane(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK:  store <vscale x 4 x i32> %[[VEC_VAL:.*]], ptr
 ; CHECK: middle.block:
 ; CHECK: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECK-NEXT: %[[VSCALE2:.*]] = shl nuw i32 %[[VSCALE]], 2
 ; CHECK-NEXT: %[[LAST_LANE:.*]] = add i32 %[[VSCALE2]], -1
 ; CHECK-NEXT: %{{.*}} = extractelement <vscale x 4 x i32> %[[VEC_VAL]], i32 %[[LAST_LANE]]
 
@@ -39,7 +39,7 @@ define float @ret_last_lane(ptr noalias nocapture %a, ptr noalias nocapture read
 ; CHECK:  store <vscale x 4 x float> %[[VEC_VAL:.*]], ptr
 ; CHECK: middle.block:
 ; CHECK: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT: %[[VSCALE2:.*]] = shl i32 %[[VSCALE]], 2
+; CHECK-NEXT: %[[VSCALE2:.*]] = shl nuw i32 %[[VSCALE]], 2
 ; CHECK-NEXT: %[[LAST_LANE:.*]] = add i32 %[[VSCALE2]], -1
 ; CHECK-NEXT: %{{.*}} = extractelement <vscale x 4 x float> %[[VEC_VAL]], i32 %[[LAST_LANE]]
 
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
index cfb96b4f5a61..0322f74ac343 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll
@@ -16,30 +16,30 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[WIDE_TRIP_COUNT]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[D1]], [[S2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 16
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i32 0
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i64 [[TMP14]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x half>, ptr [[TMP12]], align 2
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 8 x half>, ptr [[TMP15]], align 2
@@ -48,7 +48,7 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read
 ; CHECK-NEXT:    [[TMP18:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i32 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 8
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 8
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 8 x half> [[TMP16]], ptr [[TMP19]], align 2
 ; CHECK-NEXT:    store <vscale x 8 x half> [[TMP17]], ptr [[TMP22]], align 2
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
index f44744071ae5..70042caaf961 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll
@@ -13,11 +13,11 @@ define void @induction_i7(ptr %dst) #0 {
 ; CHECK-SAME: ptr [[DST:%.*]])
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i7
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP40]], i64 0
@@ -39,7 +39,7 @@ define void @induction_i7(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP24:%.*]] = zext <vscale x 2 x i7> [[TMP20]] to <vscale x 2 x i64>
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
 ; CHECK-NEXT:    [[TMP26:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP27:%.*]] = mul i64 [[TMP26]], 2
+; CHECK-NEXT:    [[TMP27:%.*]] = mul nuw i64 [[TMP26]], 2
 ; CHECK-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP27]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP23]], ptr [[TMP25]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP24]], ptr [[TMP28]], align 8
@@ -78,11 +78,11 @@ define void @induction_i3_zext(ptr %dst) #0 {
 ; CHECK-SAME: ptr [[DST:%.*]])
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 64, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 64, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP40:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP40:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP40]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = trunc i64 [[N_VEC]] to i3
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP40]], i64 0
@@ -102,7 +102,7 @@ define void @induction_i3_zext(ptr %dst) #0 {
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = mul i64 [[TMP24]], 2
+; CHECK-NEXT:    [[TMP25:%.*]] = mul nuw i64 [[TMP24]], 2
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i64 [[TMP25]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP19]], ptr [[TMP23]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP20]], ptr [[TMP26]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
index 2f90b5a332bd..4f0637fd8db2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll
@@ -15,16 +15,16 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly
 ; CHECK-LABEL: @cond_ind64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 7e4edf739695..8c2958769a61 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1369,16 +1369,15 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
 ; CHECK-LABEL: @interleave_deinterleave_factor3(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], 1024
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 256
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP5]], i64 0
 ; CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -1464,16 +1463,15 @@ define void @interleave_deinterleave(ptr writeonly noalias %dst, ptr readonly %a
 ; CHECK-LABEL: @interleave_deinterleave(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
-; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP1]], 1024
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i64 [[TMP0]], 256
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
index bd2bd5aa2795..f152dd308cb6 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll
@@ -27,16 +27,15 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -78,10 +77,12 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
@@ -172,16 +173,15 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -216,10 +216,12 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
@@ -299,16 +301,15 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD1]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV3]], i64 0
@@ -347,10 +348,12 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD1]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
@@ -446,16 +449,15 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; SCALAR_TAIL_FOLDING-NEXT:  entry:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_TAIL_FOLDING-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_TAIL_FOLDING-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_TAIL_FOLDING:       vector.ph:
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_TAIL_FOLDING-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_TAIL_FOLDING-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -501,10 +503,12 @@ define dso_local void @masked_strided_factor4(ptr noalias nocapture readonly %p,
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP3]])
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPA:%.*]] = sub i32 1024, [[TMP3]]
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMPB:%.*]] = icmp ult i32 [[TMP2]], 64
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = select i1 [[TMPB]], i32 [[TMPA]], i32 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 16 x i1> @llvm.get.active.lane.mask.nxv16i1.i32(i32 0, i32 1024)
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
index 8b009f1c9137..f8c635baf13c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll
@@ -7,16 +7,16 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N)
 ; CHECK-LABEL: @inv_store_i16(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -24,7 +24,7 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N)
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP8]], align 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP11]]
 ; CHECK-NEXT:    store i16 [[TMP12]], ptr [[DST:%.*]], align 2
@@ -57,16 +57,16 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64
 ; CHECK-LABEL: @cond_inv_store_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
index be1ff0710a41..cac526f16213 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll
@@ -11,16 +11,16 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[TMP2:%.*]] = lshr i64 [[TMP1]], 3
 ; CHECK-NEXT:    [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[N_VEC]], 8
@@ -32,7 +32,7 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) {
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[OFFSET_IDX]]
 ; CHECK-NEXT:    [[TMP32:%.*]] = getelementptr i64, ptr [[TMP30]], i32 0
 ; CHECK-NEXT:    [[TMP33:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP34:%.*]] = mul i64 [[TMP33]], 2
+; CHECK-NEXT:    [[TMP34:%.*]] = mul nuw i64 [[TMP33]], 2
 ; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr i64, ptr [[TMP30]], i64 [[TMP34]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP32]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> zeroinitializer, ptr [[TMP35]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
index ca2f969552a2..ce7b78ecc248 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll
@@ -5,9 +5,9 @@ target triple = "aarch64-unknown-linux-gnu"
 define void @trip7_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 {
 ; CHECK-LABEL: @trip7_i64(
 ; CHECK:         = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    = mul i64
+; CHECK-NEXT:    = mul nuw i64
 ; CHECK:         [[VSCALE:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[VF:%.*]] = mul i64 [[VSCALE]], 2
+; CHECK-NEXT:    [[VF:%.*]] = mul nuw i64 [[VSCALE]], 2
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ]
 ; CHECK:         [[ACTIVE_LANE_MASK:%.*]] = phi <vscale x 2 x i1> [ {{%.*}}, %vector.ph ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %vector.body ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
index bc4533f3011c..246beb297cd2 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll
@@ -16,39 +16,39 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[N:%.*]], i32 999)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0
 ; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP21]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP27]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD3]], ptr [[TMP30]], align 4
@@ -90,39 +90,39 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 {
 ; CHECK-NEXT:    [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[N:%.*]], i32 999)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add nuw nsw i32 [[UMIN]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[B1]], [[A2]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 8
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[TMP0]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 8
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0
 ; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; CHECK-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
 ; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP21]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x i32>, ptr [[TMP24]], align 4
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]]
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD]], ptr [[TMP27]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x i32> [[WIDE_LOAD3]], ptr [[TMP30]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
index 18d05d2a5b54..b6f723e049be 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll
@@ -14,13 +14,13 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[DST_21:%.*]] = ptrtoint ptr [[DST_2:%.*]] to i64
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 20, i64 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[DST_21]], [[DST_12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -43,11 +43,11 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX11]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP16]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP49:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP50:%.*]] = mul i64 [[TMP49]], 4
+; CHECK-NEXT:    [[TMP50:%.*]] = mul nuw i64 [[TMP49]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -55,13 +55,13 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr i64, ptr [[SRC_2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr i64, ptr [[TMP23]], i32 0
 ; CHECK-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 2
+; CHECK-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 2
 ; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr i64, ptr [[TMP23]], i64 [[TMP29]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 2 x i64>, ptr [[TMP27]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <vscale x 2 x i64>, ptr [[TMP30]], align 8
 ; CHECK-NEXT:    [[TMP31:%.*]] = getelementptr i64, ptr [[TMP25]], i32 0
 ; CHECK-NEXT:    [[TMP32:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP33:%.*]] = mul i64 [[TMP32]], 2
+; CHECK-NEXT:    [[TMP33:%.*]] = mul nuw i64 [[TMP32]], 2
 ; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr i64, ptr [[TMP25]], i64 [[TMP33]]
 ; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <vscale x 2 x i64>, ptr [[TMP31]], align 8
 ; CHECK-NEXT:    [[WIDE_LOAD14:%.*]] = load <vscale x 2 x i64>, ptr [[TMP34]], align 8
@@ -71,13 +71,13 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr
 ; CHECK-NEXT:    [[TMP39:%.*]] = getelementptr i64, ptr [[DST_2]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP41:%.*]] = getelementptr i64, ptr [[TMP37]], i32 0
 ; CHECK-NEXT:    [[TMP42:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP43:%.*]] = mul i64 [[TMP42]], 2
+; CHECK-NEXT:    [[TMP43:%.*]] = mul nuw i64 [[TMP42]], 2
 ; CHECK-NEXT:    [[TMP44:%.*]] = getelementptr i64, ptr [[TMP37]], i64 [[TMP43]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP35]], ptr [[TMP41]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP36]], ptr [[TMP44]], align 8
 ; CHECK-NEXT:    [[TMP45:%.*]] = getelementptr i64, ptr [[TMP39]], i32 0
 ; CHECK-NEXT:    [[TMP46:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP47:%.*]] = mul i64 [[TMP46]], 2
+; CHECK-NEXT:    [[TMP47:%.*]] = mul nuw i64 [[TMP46]], 2
 ; CHECK-NEXT:    [[TMP48:%.*]] = getelementptr i64, ptr [[TMP39]], i64 [[TMP47]]
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP35]], ptr [[TMP45]], align 8
 ; CHECK-NEXT:    store <vscale x 2 x i64> [[TMP36]], ptr [[TMP48]], align 8
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
index 8fbb356c7974..eb8f218f9938 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll
@@ -47,15 +47,15 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 4
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
index 5a4e0efb36cb..90b490148be8 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll
@@ -9,13 +9,13 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025)
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
index f4639203d103..2de24b0f654d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll
@@ -13,15 +13,15 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -67,15 +67,15 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
 ; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -139,15 +139,15 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -192,15 +192,15 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 {
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
 ; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -262,15 +262,15 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 4
+; CHECK-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -329,15 +329,15 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 {
 ; CHECK-IN-LOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK-IN-LOOP:       vector.ph:
 ; CHECK-IN-LOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-IN-LOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]]
 ; CHECK-IN-LOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-IN-LOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-IN-LOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-IN-LOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-IN-LOOP-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
index 977115ce5321..ea9cd3f5d854 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll
@@ -11,26 +11,26 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP61:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP62:%.*]] = mul i64 [[TMP61]], 16
+; CHECK-NEXT:    [[TMP62:%.*]] = mul nuw i64 [[TMP61]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
+; CHECK-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
 ; CHECK-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP28]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 12
+; CHECK-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 12
 ; CHECK-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP30]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[UMAX]])
@@ -48,13 +48,13 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr i32, ptr [[TMP47]], i32 0
 ; CHECK-NEXT:    [[TMP52:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP53:%.*]] = mul i64 [[TMP52]], 4
+; CHECK-NEXT:    [[TMP53:%.*]] = mul nuw i64 [[TMP52]], 4
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP56:%.*]] = mul i64 [[TMP55]], 8
+; CHECK-NEXT:    [[TMP56:%.*]] = mul nuw i64 [[TMP55]], 8
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP59:%.*]] = mul i64 [[TMP58]], 12
+; CHECK-NEXT:    [[TMP59:%.*]] = mul nuw i64 [[TMP58]], 12
 ; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP59]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP51]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]])
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP54]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK7]])
@@ -62,13 +62,13 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP60]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK9]])
 ; CHECK-NEXT:    [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]]
 ; CHECK-NEXT:    [[TMP63:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP64:%.*]] = mul i64 [[TMP63]], 4
+; CHECK-NEXT:    [[TMP64:%.*]] = mul nuw i64 [[TMP63]], 4
 ; CHECK-NEXT:    [[TMP65:%.*]] = add i64 [[INDEX6]], [[TMP64]]
 ; CHECK-NEXT:    [[TMP66:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP67:%.*]] = mul i64 [[TMP66]], 8
+; CHECK-NEXT:    [[TMP67:%.*]] = mul nuw i64 [[TMP66]], 8
 ; CHECK-NEXT:    [[TMP68:%.*]] = add i64 [[INDEX6]], [[TMP67]]
 ; CHECK-NEXT:    [[TMP69:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP70:%.*]] = mul i64 [[TMP69]], 12
+; CHECK-NEXT:    [[TMP70:%.*]] = mul nuw i64 [[TMP69]], 12
 ; CHECK-NEXT:    [[TMP71:%.*]] = add i64 [[INDEX6]], [[TMP70]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT11]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP65]], i64 [[TMP9]])
@@ -103,26 +103,26 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP83:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP84:%.*]] = mul i64 [[TMP83]], 16
+; CHECK-NEXT:    [[TMP84:%.*]] = mul nuw i64 [[TMP83]], 16
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
 ; CHECK-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; CHECK-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; CHECK-NEXT:    [[INDEX_PART_NEXT:%.*]] = add i64 0, [[TMP26]]
 ; CHECK-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 8
+; CHECK-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 8
 ; CHECK-NEXT:    [[INDEX_PART_NEXT1:%.*]] = add i64 0, [[TMP28]]
 ; CHECK-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 12
+; CHECK-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 12
 ; CHECK-NEXT:    [[INDEX_PART_NEXT2:%.*]] = add i64 0, [[TMP30]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY3:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_PART_NEXT]], i64 [[UMAX]])
@@ -140,13 +140,13 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[TMP47:%.*]] = getelementptr i32, ptr [[COND_PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = getelementptr i32, ptr [[TMP47]], i32 0
 ; CHECK-NEXT:    [[TMP52:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP53:%.*]] = mul i64 [[TMP52]], 4
+; CHECK-NEXT:    [[TMP53:%.*]] = mul nuw i64 [[TMP52]], 4
 ; CHECK-NEXT:    [[TMP54:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP53]]
 ; CHECK-NEXT:    [[TMP55:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP56:%.*]] = mul i64 [[TMP55]], 8
+; CHECK-NEXT:    [[TMP56:%.*]] = mul nuw i64 [[TMP55]], 8
 ; CHECK-NEXT:    [[TMP57:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP56]]
 ; CHECK-NEXT:    [[TMP58:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP59:%.*]] = mul i64 [[TMP58]], 12
+; CHECK-NEXT:    [[TMP59:%.*]] = mul nuw i64 [[TMP58]], 12
 ; CHECK-NEXT:    [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP59]]
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP51]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK]], <vscale x 4 x i32> poison)
 ; CHECK-NEXT:    [[WIDE_MASKED_LOAD10:%.*]] = call <vscale x 4 x i32> @llvm.masked.load.nxv4i32.p0(ptr [[TMP54]], i32 4, <vscale x 4 x i1> [[ACTIVE_LANE_MASK7]], <vscale x 4 x i32> poison)
@@ -163,13 +163,13 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    [[TMP65:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX6]]
 ; CHECK-NEXT:    [[TMP73:%.*]] = getelementptr i32, ptr [[TMP65]], i32 0
 ; CHECK-NEXT:    [[TMP74:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP75:%.*]] = mul i64 [[TMP74]], 4
+; CHECK-NEXT:    [[TMP75:%.*]] = mul nuw i64 [[TMP74]], 4
 ; CHECK-NEXT:    [[TMP76:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP75]]
 ; CHECK-NEXT:    [[TMP77:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP78:%.*]] = mul i64 [[TMP77]], 8
+; CHECK-NEXT:    [[TMP78:%.*]] = mul nuw i64 [[TMP77]], 8
 ; CHECK-NEXT:    [[TMP79:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP78]]
 ; CHECK-NEXT:    [[TMP80:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP81:%.*]] = mul i64 [[TMP80]], 12
+; CHECK-NEXT:    [[TMP81:%.*]] = mul nuw i64 [[TMP80]], 12
 ; CHECK-NEXT:    [[TMP82:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP81]]
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP73]], i32 4, <vscale x 4 x i1> [[TMP69]])
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP76]], i32 4, <vscale x 4 x i1> [[TMP70]])
@@ -177,13 +177,13 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias %
 ; CHECK-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[BROADCAST_SPLAT]], ptr [[TMP82]], i32 4, <vscale x 4 x i1> [[TMP72]])
 ; CHECK-NEXT:    [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP84]]
 ; CHECK-NEXT:    [[TMP85:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP86:%.*]] = mul i64 [[TMP85]], 4
+; CHECK-NEXT:    [[TMP86:%.*]] = mul nuw i64 [[TMP85]], 4
 ; CHECK-NEXT:    [[TMP87:%.*]] = add i64 [[INDEX6]], [[TMP86]]
 ; CHECK-NEXT:    [[TMP88:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP89:%.*]] = mul i64 [[TMP88]], 8
+; CHECK-NEXT:    [[TMP89:%.*]] = mul nuw i64 [[TMP88]], 8
 ; CHECK-NEXT:    [[TMP90:%.*]] = add i64 [[INDEX6]], [[TMP89]]
 ; CHECK-NEXT:    [[TMP91:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP92:%.*]] = mul i64 [[TMP91]], 12
+; CHECK-NEXT:    [[TMP92:%.*]] = mul nuw i64 [[TMP91]], 12
 ; CHECK-NEXT:    [[TMP93:%.*]] = add i64 [[INDEX6]], [[TMP92]]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX6]], i64 [[TMP9]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_NEXT14]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[TMP87]], i64 [[TMP9]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
index 46dcf23ee9cf..f6f8895c2c70 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll
@@ -11,15 +11,15 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -112,15 +112,15 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -172,15 +172,15 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = sub i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP2]], [[TMP5]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = icmp ugt i64 [[TMP2]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = select i1 [[TMP11]], i64 [[TMP10]], i64 0
@@ -235,15 +235,15 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -296,15 +296,15 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) #
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -356,15 +356,15 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -430,15 +430,15 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n)
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -487,15 +487,15 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -549,15 +549,15 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
@@ -607,15 +607,15 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 {
 ; CHECK-LABEL: @simple_memset_trip1024(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
index 592dc1c4efd4..33fa3607730d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll
@@ -15,16 +15,16 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{
 ; CHECK-NEXT:    br i1 [[CMP7]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 3
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 3
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
@@ -88,7 +88,7 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    br i1 [[CMP8]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; CHECK:       for.body.preheader:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; CHECK:       vector.memcheck:
@@ -99,11 +99,11 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 {
 ; CHECK-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP7]], 3
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP7]], 3
 ; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP7]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
index 685516a57680..e2c7469a9781 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vscale-based-trip-counts.ll
@@ -10,37 +10,37 @@ define void @vscale_mul_4(ptr noalias noundef readonly captures(none) %a, ptr no
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP1]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i32 0
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP18]], 4
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    [[TMP26:%.*]] = getelementptr inbounds nuw float, ptr [[TMP13]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP14]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP26]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP27:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP16]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP17]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP27]], align 4
 ; CHECK-NEXT:    [[TMP19:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP28:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD4]]
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP19]], ptr [[TMP17]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP28]], ptr [[TMP22]], align 4
@@ -95,22 +95,22 @@ define  void @vscale_mul_8(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = shl nuw nsw i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
@@ -118,7 +118,7 @@ define  void @vscale_mul_8(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i32 0
 ; CHECK-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-NEXT:    [[TMP22:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[TMP21]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP19]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP22]], align 4
@@ -166,37 +166,37 @@ define void @vscale_mul_12(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 12
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP25:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD4]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP20]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP25]], ptr [[TMP21]], align 4
@@ -251,37 +251,37 @@ define void @vscale_mul_31(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 31
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP20]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP21]], align 4
@@ -336,37 +336,37 @@ define void @vscale_mul_64(ptr noalias noundef readonly captures(none) %a, ptr n
 ; CHECK-NEXT:    [[TMP0:%.*]] = tail call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[MUL1:%.*]] = mul nuw nsw i64 [[TMP0]], 64
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[MUL1]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[MUL1]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[MUL1]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds nuw float, ptr [[A]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i32 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds nuw float, ptr [[TMP7]], i64 [[TMP10]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x float>, ptr [[TMP8]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <vscale x 4 x float>, ptr [[TMP11]], align 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw float, ptr [[B]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i32 0
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP15]]
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 4 x float>, ptr [[TMP13]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD3:%.*]] = load <vscale x 4 x float>, ptr [[TMP16]], align 4
 ; CHECK-NEXT:    [[TMP17:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD]], [[WIDE_LOAD2]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = fmul <vscale x 4 x float> [[WIDE_LOAD1]], [[WIDE_LOAD3]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP20:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr inbounds nuw float, ptr [[TMP12]], i64 [[TMP20]]
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP17]], ptr [[TMP13]], align 4
 ; CHECK-NEXT:    store <vscale x 4 x float> [[TMP18]], ptr [[TMP21]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
index 166bdc7931cf..e58ea655d609 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll
@@ -50,16 +50,16 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-LABEL: @pointer_induction_used_as_vector(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]]
 ; CHECK-NEXT:    [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]]
@@ -68,7 +68,7 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 1, [[TMP6]]
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP12]], i64 0
@@ -144,23 +144,23 @@ define void @pointer_induction(ptr noalias %start, i64 %N) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[N:%.*]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX2:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 1, [[TMP6]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP8]], 0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[TMP11]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
index 54ba0a8c4d6b..5848d317ff14 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-epilogue.ll
@@ -9,21 +9,21 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-SAME: ptr noalias [[BUCKETS:%.*]], ptr readonly [[INDICES:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  iter.check:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.main.loop.iter.check:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP6]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP6]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK1:%.*]] = icmp ult i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK1]], label [[VEC_EPILOG_PH:%.*]], label [[VECTOR_PH1:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH1]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -40,17 +40,17 @@ define void @simple_histogram(ptr noalias %buckets, ptr readonly %indices, i64 %
 ; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_EXIT:%.*]], label [[VEC_EPILOG_ITER_CHECK:%.*]]
 ; CHECK:       vec.epilog.iter.check:
 ; CHECK-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP23:%.*]] = shl i64 [[TMP22]], 1
+; CHECK-NEXT:    [[TMP23:%.*]] = shl nuw i64 [[TMP22]], 1
 ; CHECK-NEXT:    [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], [[TMP23]]
 ; CHECK-NEXT:    br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[SCALAR_PH]], label [[VEC_EPILOG_PH]]
 ; CHECK:       vec.epilog.ph:
 ; CHECK-NEXT:    [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_PH]] ]
 ; CHECK-NEXT:    [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP25:%.*]] = shl i64 [[TMP24]], 1
+; CHECK-NEXT:    [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 1
 ; CHECK-NEXT:    [[N_MOD_VF2:%.*]] = urem i64 [[N]], [[TMP25]]
 ; CHECK-NEXT:    [[N_VEC3:%.*]] = sub i64 [[N]], [[N_MOD_VF2]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = shl i64 [[TMP16]], 1
+; CHECK-NEXT:    [[TMP17:%.*]] = shl nuw i64 [[TMP16]], 1
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
 ; CHECK:       vec.epilog.vector.body:
 ; CHECK-NEXT:    [[INDEX4:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[FOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
index b292e4304673..abee8b9340ca 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve2-histcnt-too-many-deps.ll
@@ -44,7 +44,7 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-SAME: ptr noalias [[BUCKETS:%.*]], ptr [[ARRAY:%.*]], ptr [[INDICES:%.*]], ptr [[OTHER:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NORMAL_DEP_LIMIT-NEXT:  entry:
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 8)
 ; NORMAL_DEP_LIMIT-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NORMAL_DEP_LIMIT-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -67,11 +67,11 @@ define void @many_deps(ptr noalias %buckets, ptr %array, ptr %indices, ptr %othe
 ; NORMAL_DEP_LIMIT-NEXT:    br i1 [[CONFLICT_RDX9]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
 ; NORMAL_DEP_LIMIT:       vector.ph:
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP4]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP4]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP8]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NORMAL_DEP_LIMIT-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; NORMAL_DEP_LIMIT-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NORMAL_DEP_LIMIT-NEXT:    [[TMP9:%.*]] = trunc i64 [[TMP6]] to i32
 ; NORMAL_DEP_LIMIT-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[TMP9]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
index 71d03afa6b6f..1cdc290757dd 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
@@ -14,16 +14,16 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; NONE-NEXT:  entry:
 ; NONE-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; NONE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NONE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NONE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NONE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[UMAX]], [[TMP1]]
 ; NONE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NONE:       vector.ph:
 ; NONE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NONE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NONE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NONE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]]
 ; NONE-NEXT:    [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]]
 ; NONE-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NONE-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; NONE-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NONE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; NONE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NONE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -56,18 +56,18 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; DATA-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
 ; DATA-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; DATA-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; DATA-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DATA-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA:       vector.ph:
 ; DATA-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DATA-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; DATA-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; DATA-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
 ; DATA-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
 ; DATA-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; DATA-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; DATA-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DATA-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; DATA-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; DATA-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; DATA-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; DATA-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -100,19 +100,19 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_NO_LANEMASK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; DATA_NO_LANEMASK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
 ; DATA_NO_LANEMASK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_NO_LANEMASK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; DATA_NO_LANEMASK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; DATA_NO_LANEMASK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DATA_NO_LANEMASK-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA_NO_LANEMASK:       vector.ph:
 ; DATA_NO_LANEMASK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_NO_LANEMASK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; DATA_NO_LANEMASK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; DATA_NO_LANEMASK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
 ; DATA_NO_LANEMASK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
 ; DATA_NO_LANEMASK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; DATA_NO_LANEMASK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; DATA_NO_LANEMASK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX]], 1
 ; DATA_NO_LANEMASK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_NO_LANEMASK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; DATA_NO_LANEMASK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; DATA_NO_LANEMASK-NEXT:    [[BROADCAST_SPLATINSERT5:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; DATA_NO_LANEMASK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT5]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; DATA_NO_LANEMASK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
@@ -152,18 +152,18 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
 ; DATA_AND_CONTROL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
 ; DATA_AND_CONTROL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; DATA_AND_CONTROL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; DATA_AND_CONTROL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; DATA_AND_CONTROL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA_AND_CONTROL:       vector.ph:
 ; DATA_AND_CONTROL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; DATA_AND_CONTROL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; DATA_AND_CONTROL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
 ; DATA_AND_CONTROL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]]
 ; DATA_AND_CONTROL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; DATA_AND_CONTROL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; DATA_AND_CONTROL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; DATA_AND_CONTROL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; DATA_AND_CONTROL-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]])
 ; DATA_AND_CONTROL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; DATA_AND_CONTROL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -200,15 +200,15 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA_AND_CONTROL_NO_RT_CHECK:       vector.ph:
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 4
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP8:%.*]] = icmp ugt i64 [[UMAX]], [[TMP6]]
 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT:    [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP7]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
index ce8492cd7736..e9de5e21228f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll
@@ -11,9 +11,9 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
 ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -36,12 +36,12 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; INTERLEAVE-NEXT:  entry:
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; INTERLEAVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; INTERLEAVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
 ; INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 1
+; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 [[N]])
 ; INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -65,7 +65,7 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6
 ; INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP11]], ptr [[TMP14]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; INTERLEAVE-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP16:%.*]] = shl i64 [[TMP15]], 1
+; INTERLEAVE-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 1
 ; INTERLEAVE-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]]
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]])
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 [[TMP4]])
@@ -97,9 +97,9 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
 ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -122,12 +122,12 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
 ; INTERLEAVE-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; INTERLEAVE-NEXT:  entry:
 ; INTERLEAVE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; INTERLEAVE-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; INTERLEAVE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; INTERLEAVE-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; INTERLEAVE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP3]])
 ; INTERLEAVE-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 1
+; INTERLEAVE-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 1
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 [[N]])
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_ENTRY1:%.*]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP6]], i64 [[N]])
 ; INTERLEAVE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -151,7 +151,7 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3
 ; INTERLEAVE-NEXT:    call void @llvm.masked.store.nxv2f64.p0(<vscale x 2 x double> [[TMP11]], ptr [[TMP14]], i32 8, <vscale x 2 x i1> [[ACTIVE_LANE_MASK2]])
 ; INTERLEAVE-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP1]]
 ; INTERLEAVE-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; INTERLEAVE-NEXT:    [[TMP16:%.*]] = shl i64 [[TMP15]], 1
+; INTERLEAVE-NEXT:    [[TMP16:%.*]] = shl nuw i64 [[TMP15]], 1
 ; INTERLEAVE-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], [[TMP16]]
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP4]])
 ; INTERLEAVE-NEXT:    [[ACTIVE_LANE_MASK_NEXT4]] = call <vscale x 2 x i1> @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP17]], i64 [[TMP4]])
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
index f178805608eb..3545c6b2239d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll
@@ -10,11 +10,11 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 {
 ; WIDE-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; WIDE:       vector.ph:
 ; WIDE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; WIDE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; WIDE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; WIDE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; WIDE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; WIDE-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; WIDE-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; WIDE-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; WIDE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; WIDE:       vector.body:
 ; WIDE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
index 2587b932dd76..2b4d8b99847d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/bf16.ll
@@ -25,16 +25,16 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; ZVFBFMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ZVFBFMIN-NEXT:  [[ENTRY:.*]]:
 ; ZVFBFMIN-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
+; ZVFBFMIN-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
 ; ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP8]]
 ; ZVFBFMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFBFMIN:       [[VECTOR_PH]]:
 ; ZVFBFMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; ZVFBFMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; ZVFBFMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; ZVFBFMIN-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP12]], 8
+; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 8
 ; ZVFBFMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFBFMIN:       [[VECTOR_BODY]]:
 ; ZVFBFMIN-NEXT:    [[TMP0:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -142,16 +142,16 @@ define void @vfwmaccbf16.vv(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64
 ; ZVFBFMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; ZVFBFMIN-NEXT:  [[ENTRY:.*]]:
 ; ZVFBFMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVFBFMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVFBFMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; ZVFBFMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFBFMIN:       [[VECTOR_PH]]:
 ; ZVFBFMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVFBFMIN-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVFBFMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; ZVFBFMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; ZVFBFMIN-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVFBFMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVFBFMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFBFMIN:       [[VECTOR_BODY]]:
 ; ZVFBFMIN-NEXT:    [[TMP6:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
index 362cf093f180..75ae6df5fcd3 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blend-any-of-reduction-cost.ll
@@ -63,16 +63,16 @@ define i32 @any_of_reduction_used_in_blend_with_multiple_phis(ptr %src, i64 %N,
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_1]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 2 x i1> [[BROADCAST_SPLATINSERT1]], <vscale x 2 x i1> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i1> poison, i1 [[C_0]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
index d41caca97e1f..f8b83ff41f51 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/blocks-with-dead-instructions.ll
@@ -12,18 +12,18 @@ define void @block_with_dead_inst_1(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
@@ -97,18 +97,18 @@ define void @block_with_dead_inst_2(ptr %src) #0 {
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
@@ -182,18 +182,18 @@ define void @multiple_blocks_with_dead_insts_3(ptr %src) #0 {
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
@@ -277,18 +277,18 @@ define void @multiple_blocks_with_dead_insts_4(ptr %src, i64 %N) #0 {
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
@@ -374,18 +374,18 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_5(ptr %src) #0 {
 ; CHECK-SAME: ptr [[SRC:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 333, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 333, [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 333, [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 3)
@@ -479,18 +479,18 @@ define void @multiple_blocks_with_dead_inst_multiple_successors_6(ptr %src, i1 %
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[TMP0]], 3
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i64 [[TMP6]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[TMP8]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP11:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul <vscale x 8 x i64> [[TMP11]], splat (i64 3)
@@ -581,16 +581,16 @@ define void @empty_block_with_phi_1(ptr %src, i64 %N) #0 {
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -660,16 +660,16 @@ define void @empty_block_with_phi_2(ptr %src, i64 %N) #0 {
 ; CHECK-SAME: ptr [[SRC:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -746,7 +746,7 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[UMIN7]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.umax.i64(i64 40, i64 [[TMP5]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP6]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -773,11 +773,11 @@ define void @dead_load_in_block(ptr %dst, ptr %src, i8 %N, i64 %x) #0 {
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 2
+; CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP15]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP17:%.*]] = mul i64 [[TMP16]], 2
+; CHECK-NEXT:    [[TMP17:%.*]] = mul nuw i64 [[TMP16]], 2
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[TMP18:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    [[TMP20:%.*]] = mul <vscale x 2 x i64> [[TMP18]], splat (i64 3)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
index 4450678871ac..22c56c89fa16 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/dead-ops-cost.ll
@@ -19,18 +19,18 @@ define void @dead_load(ptr %p, i16 %start) {
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[UMIN]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], [[TMP7]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP5]], [[TMP9]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = select i1 [[TMP10]], i64 [[TMP9]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP11]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 8
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[START_EXT]], [[TMP18]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = call <vscale x 8 x i64> @llvm.stepvector.nxv8i64()
@@ -93,7 +93,7 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DST:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.umax.i32(i32 8, i32 [[TMP1]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i32 252, [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -106,13 +106,13 @@ define i8 @dead_live_out_due_to_scalar_epilogue_required(ptr %src, ptr %dst) {
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 252, [[TMP4]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP6:%.*]] = select i1 [[TMP5]], i32 [[TMP4]], i32 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 252, [[TMP6]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i32 [[N_VEC]], 4
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i32> [[TMP9]], splat (i32 4)
@@ -317,16 +317,16 @@ define void @test_phi_in_latch_redundant(ptr %dst, i32 %a) {
 ; CHECK-SAME: ptr [[DST:%.*]], i32 [[A:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 37, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 37, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 37, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[N_VEC]], 9
@@ -402,16 +402,16 @@ define void @gather_interleave_group_with_dead_insert_pos(i64 %N, ptr noalias %s
 ; CHECK-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[N_VEC]], 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP11:%.*]] = mul <vscale x 4 x i64> [[TMP9]], splat (i64 2)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
index eccba717f747..a2faaaaf06df 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll
@@ -14,16 +14,16 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v) {
 ; CHECK-LABEL: @vector_add(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -76,16 +76,16 @@ define i64 @vector_add_reduce(ptr noalias nocapture %a) {
 ; CHECK-LABEL: @vector_add_reduce(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
index 73a71f7557f3..db780c3c12c7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll
@@ -11,16 +11,16 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_udiv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -111,16 +111,16 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_sdiv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -211,16 +211,16 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_urem(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -311,16 +311,16 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @vector_srem(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -411,16 +411,16 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @predicated_udiv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
@@ -536,16 +536,16 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = icmp ne <vscale x 2 x i64> [[BROADCAST_SPLAT]], zeroinitializer
@@ -661,16 +661,16 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_udiv_by_constant(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -781,16 +781,16 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv_by_constant(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -901,16 +901,16 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @predicated_sdiv_by_minus_one(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll b/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
index b8c030c0c60b..0b307c28cecc 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/f16.ll
@@ -25,16 +25,16 @@ define void @fadd(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; ZVFHMIN-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ZVFHMIN-NEXT:  [[ENTRY:.*]]:
 ; ZVFHMIN-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
+; ZVFHMIN-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP8]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP12]], 8
+; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP12]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
index 1319454b7a1a..283688c8e447 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/fminimumnum.ll
@@ -11,13 +11,13 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP16]]
@@ -28,11 +28,11 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -77,13 +77,13 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -94,11 +94,11 @@ define void @fmin32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -166,13 +166,13 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP16]]
@@ -183,11 +183,11 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP19]], 4
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 4
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -232,13 +232,13 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -249,11 +249,11 @@ define void @fmax32(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -321,13 +321,13 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP16]]
@@ -338,11 +338,11 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP19]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -387,13 +387,13 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -404,11 +404,11 @@ define void @fmin64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -476,13 +476,13 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP15]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP15]], 2
 ; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP16]]
@@ -493,11 +493,11 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP9]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i64 [[TMP9]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP18]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP19]], 2
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP19]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -542,13 +542,13 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -559,11 +559,11 @@ define void @fmax64(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 2
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -631,13 +631,13 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP18]]
@@ -648,11 +648,11 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -697,13 +697,13 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -714,11 +714,11 @@ define void @fmin16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -786,13 +786,13 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; CHECK-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    [[TMP14:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP13]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP14]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 8
+; CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 8
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP16]], 2
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; CHECK-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP18]]
@@ -803,11 +803,11 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -852,13 +852,13 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    [[INPUT12:%.*]] = ptrtoint ptr [[INPUT1]] to i64
 ; ZVFHMIN-NEXT:    [[OUTPUT1:%.*]] = ptrtoint ptr [[OUTPUT]] to i64
 ; ZVFHMIN-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; ZVFHMIN-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; ZVFHMIN-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; ZVFHMIN-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 4096, [[TMP2]]
 ; ZVFHMIN-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; ZVFHMIN:       [[VECTOR_MEMCHECK]]:
 ; ZVFHMIN-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; ZVFHMIN-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; ZVFHMIN-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
 ; ZVFHMIN-NEXT:    [[TMP6:%.*]] = sub i64 [[OUTPUT1]], [[INPUT12]]
 ; ZVFHMIN-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
@@ -869,11 +869,11 @@ define void @fmax16(ptr noundef readonly captures(none) %input1, ptr noundef rea
 ; ZVFHMIN-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; ZVFHMIN:       [[VECTOR_PH]]:
 ; ZVFHMIN-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; ZVFHMIN-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; ZVFHMIN-NEXT:    [[N_MOD_VF:%.*]] = urem i64 4096, [[TMP10]]
 ; ZVFHMIN-NEXT:    [[N_VEC:%.*]] = sub i64 4096, [[N_MOD_VF]]
 ; ZVFHMIN-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; ZVFHMIN-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; ZVFHMIN-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; ZVFHMIN:       [[VECTOR_BODY]]:
 ; ZVFHMIN-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
index 5c15660e8713..e6825faf3f8d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/induction-costs.ll
@@ -21,7 +21,7 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[UMIN21]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.umax.i64(i64 128, i64 [[TMP7]])
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP5]], [[TMP8]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -58,13 +58,13 @@ define void @skip_free_iv_truncate(i16 %x, ptr %A) #0 {
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP45:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP46:%.*]] = mul i64 [[TMP45]], 8
+; CHECK-NEXT:    [[TMP46:%.*]] = mul nuw i64 [[TMP45]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP5]], [[TMP46]]
 ; CHECK-NEXT:    [[TMP47:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP48:%.*]] = select i1 [[TMP47]], i64 [[TMP46]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP5]], [[TMP48]]
 ; CHECK-NEXT:    [[TMP51:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP52:%.*]] = mul i64 [[TMP51]], 8
+; CHECK-NEXT:    [[TMP52:%.*]] = mul nuw i64 [[TMP51]], 8
 ; CHECK-NEXT:    [[TMP49:%.*]] = mul i64 [[N_VEC]], 3
 ; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[X_I64]], [[TMP49]]
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index 8e90287bac2a..a8e6ef7ebfec 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -15,16 +15,16 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; OUTLOOP-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; OUTLOOP:       for.body.preheader:
 ; OUTLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
 ; OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; OUTLOOP:       vector.ph:
 ; OUTLOOP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
 ; OUTLOOP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; OUTLOOP:       vector.body:
 ; OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -68,16 +68,16 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; INLOOP-NEXT:    br i1 [[CMP6]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
 ; INLOOP:       for.body.preheader:
 ; INLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 8
+; INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
 ; INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N]], [[TMP1]]
 ; INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INLOOP:       vector.ph:
 ; INLOOP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 8
+; INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 8
 ; INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]]
 ; INLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
 ; INLOOP-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; INLOOP-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 8
+; INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 8
 ; INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INLOOP:       vector.body:
 ; INLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -123,13 +123,13 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP2]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 4
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
 ; IF-EVL-OUTLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -179,13 +179,13 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) {
 ; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 8
+; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
 ; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = sub i32 [[TMP1]], 1
 ; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP2]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = mul i32 [[TMP3]], 8
+; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = mul nuw i32 [[TMP3]], 8
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -250,16 +250,16 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; OUTLOOP-LABEL: @smin(
 ; OUTLOOP-NEXT:  entry:
 ; OUTLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; OUTLOOP:       vector.ph:
 ; OUTLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; OUTLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -299,16 +299,16 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; INLOOP-LABEL: @smin(
 ; INLOOP-NEXT:  entry:
 ; INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; INLOOP:       vector.ph:
 ; INLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; INLOOP:       vector.body:
 ; INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -346,18 +346,18 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -403,18 +403,18 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
index 61a3e3561ad9..85ccbab0e670 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll
@@ -7,16 +7,16 @@ define void @load_store_factor2_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor2_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -103,16 +103,16 @@ define void @load_store_factor2_i32(ptr %p) {
 ; SCALABLE-LABEL: @load_store_factor2_i32(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -181,16 +181,16 @@ define void @load_store_factor2_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor2_i64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -277,16 +277,16 @@ define void @load_store_factor2_i64(ptr %p) {
 ; SCALABLE-LABEL: @load_store_factor2_i64(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -355,16 +355,16 @@ define void @load_store_factor3_i32(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -467,16 +467,16 @@ define void @load_store_factor3_i32(ptr %p) {
 ; SCALABLE-LABEL: @load_store_factor3_i32(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; SCALABLE-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -558,16 +558,16 @@ define void @load_store_factor3_i64(ptr %p) {
 ; CHECK-LABEL: @load_store_factor3_i64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -670,16 +670,16 @@ define void @load_store_factor3_i64(ptr %p) {
 ; SCALABLE-LABEL: @load_store_factor3_i64(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 2
+; SCALABLE-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP7]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -761,16 +761,16 @@ define void @load_store_factor4(ptr %p) {
 ; CHECK-LABEL: @load_store_factor4(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -887,16 +887,16 @@ define void @load_store_factor4(ptr %p) {
 ; SCALABLE-LABEL: @load_store_factor4(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2171,16 +2171,16 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; CHECK-LABEL: @combine_load_factor2_i32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2273,16 +2273,16 @@ define void @combine_load_factor2_i32(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-LABEL: @combine_load_factor2_i32(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2351,16 +2351,16 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; CHECK-LABEL: @combine_load_factor2_i64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2453,16 +2453,16 @@ define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) {
 ; SCALABLE-LABEL: @combine_load_factor2_i64(
 ; SCALABLE-NEXT:  entry:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
index 79425ae3a67e..c0548dd8ca54 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-masked-access.ll
@@ -11,16 +11,15 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:  entry:
 ; SCALAR_EPILOGUE-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_EPILOGUE-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
 ; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -63,12 +62,12 @@ define void @masked_strided_factor2(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -180,16 +179,15 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; SCALAR_EPILOGUE-NEXT:  entry:
 ; SCALAR_EPILOGUE-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; SCALAR_EPILOGUE-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
-; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP1]], 1024
+; SCALAR_EPILOGUE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ugt i32 [[TMP0]], 64
 ; SCALAR_EPILOGUE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; SCALAR_EPILOGUE:       vector.ph:
 ; SCALAR_EPILOGUE-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]]
 ; SCALAR_EPILOGUE-NEXT:    [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]]
 ; SCALAR_EPILOGUE-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl i32 [[TMP4]], 4
+; SCALAR_EPILOGUE-NEXT:    [[TMP5:%.*]] = shl nuw i32 [[TMP4]], 4
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; SCALAR_EPILOGUE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; SCALAR_EPILOGUE-NEXT:    [[TMP6:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
@@ -248,12 +246,12 @@ define void @masked_strided_factor4(ptr noalias nocapture readonly %p, ptr noali
 ; PREDICATED_TAIL_FOLDING:       vector.ph:
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[CONV:%.*]] = zext i8 [[GUARD]] to i32
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl i32 [[TMP0]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP1:%.*]] = shl nuw i32 [[TMP0]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_RND_UP:%.*]] = add i32 [[TMP1]], 1023
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl i32 [[TMP2]], 4
+; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP3:%.*]] = shl nuw i32 [[TMP2]], 4
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 16 x i32> poison, i32 [[CONV]], i64 0
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 16 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 16 x i32> poison, <vscale x 16 x i32> zeroinitializer
 ; PREDICATED_TAIL_FOLDING-NEXT:    [[TMP4:%.*]] = call <vscale x 16 x i32> @llvm.stepvector.nxv16i32()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
index 964615b03049..4d59f272d8b9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll
@@ -48,16 +48,16 @@ define void @load_store(ptr %p) {
 ; LMUL2-LABEL: @load_store(
 ; LMUL2-NEXT:  entry:
 ; LMUL2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; LMUL2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; LMUL2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; LMUL2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL2:       vector.ph:
 ; LMUL2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; LMUL2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; LMUL2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; LMUL2-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; LMUL2-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL2-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; LMUL2-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; LMUL2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL2:       vector.body:
 ; LMUL2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -90,16 +90,16 @@ define void @load_store(ptr %p) {
 ; LMUL4-LABEL: @load_store(
 ; LMUL4-NEXT:  entry:
 ; LMUL4-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; LMUL4-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; LMUL4-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; LMUL4-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL4:       vector.ph:
 ; LMUL4-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; LMUL4-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; LMUL4-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; LMUL4-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; LMUL4-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL4-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; LMUL4-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; LMUL4-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL4:       vector.body:
 ; LMUL4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -132,16 +132,16 @@ define void @load_store(ptr %p) {
 ; LMUL8-LABEL: @load_store(
 ; LMUL8-NEXT:  entry:
 ; LMUL8-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; LMUL8-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; LMUL8-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; LMUL8-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; LMUL8:       vector.ph:
 ; LMUL8-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; LMUL8-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; LMUL8-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; LMUL8-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; LMUL8-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; LMUL8-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 8
+; LMUL8-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 8
 ; LMUL8-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; LMUL8:       vector.body:
 ; LMUL8-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
index 17c737157a56..158022f1879a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll
@@ -126,13 +126,13 @@ define void @trip8_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 8, [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 8)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
index 3e4d337c0706..4ccc45d4daf9 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll
@@ -12,16 +12,16 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) {
 ; VLENUNK-LABEL: @test(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; VLENUNK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VLENUNK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
index 2276b592aac8..c94b2026c4f7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll
@@ -17,7 +17,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32-LABEL: @foo4(
 ; RV32-NEXT:  entry:
 ; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP2]]
 ; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -35,11 +35,11 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV32-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; RV32:       vector.ph:
 ; RV32-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; RV32-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
 ; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
 ; RV32-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV32-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; RV32-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
 ; RV32-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; RV32-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 16)
@@ -96,7 +96,7 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64-LABEL: @foo4(
 ; RV64-NEXT:  entry:
 ; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 625, [[TMP2]]
 ; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -114,11 +114,11 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea
 ; RV64-NEXT:    br i1 [[CONFLICT_RDX]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; RV64:       vector.ph:
 ; RV64-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; RV64-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]]
 ; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]]
 ; RV64-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; RV64-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; RV64-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 16
 ; RV64-NEXT:    [[TMP7:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; RV64-NEXT:    [[TMP9:%.*]] = mul <vscale x 2 x i64> [[TMP7]], splat (i64 16)
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
index 8c29da02b813..b8d5cbd5d47b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/partial-reduce-dot-product.ll
@@ -11,16 +11,16 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
 ; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -48,16 +48,16 @@ define i32 @vqdot(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
 ; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -183,16 +183,16 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
 ; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -220,16 +220,16 @@ define i32 @vqdotu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
 ; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -355,16 +355,16 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
 ; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -392,16 +392,16 @@ define i32 @vqdotsu(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
 ; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -526,16 +526,16 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; V-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; V-NEXT:  entry:
 ; V-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; V-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; V-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; V-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; V:       vector.ph:
 ; V-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; V-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; V-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; V-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; V-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; V-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; V-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; V-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; V:       vector.body:
 ; V-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -563,16 +563,16 @@ define i32 @vqdotsu2(ptr %a, ptr %b) #0 {
 ; ZVQDOTQ-SAME: ptr [[A:%.*]], ptr [[B:%.*]]) #[[ATTR0]] {
 ; ZVQDOTQ-NEXT:  entry:
 ; ZVQDOTQ-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; ZVQDOTQ-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; ZVQDOTQ-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; ZVQDOTQ-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; ZVQDOTQ:       vector.ph:
 ; ZVQDOTQ-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; ZVQDOTQ-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; ZVQDOTQ-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; ZVQDOTQ-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; ZVQDOTQ-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; ZVQDOTQ-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; ZVQDOTQ-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; ZVQDOTQ:       vector.body:
 ; ZVQDOTQ-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
index 2f9ff20bf0f9..8088a6507c25 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr87378-vpinstruction-or-drop-poison-generating-flags.ll
@@ -10,16 +10,16 @@ define void @pr87378_vpinstruction_or_drop_poison_generating_flags(ptr %arg, i64
 ; CHECK-SAME: ptr [[ARG:%.*]], i64 [[A:%.*]], i64 [[B:%.*]], i64 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1001, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1001, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1001, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[A]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[B]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
index e3e727b41c02..85163c79072b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/remark-reductions.ll
@@ -6,16 +6,16 @@ define float @s311(float %a_0, float %s311_sum) {
 ; CHECK-SAME: float [[A_0:%.*]], float [[S311_SUM:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 1200, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 1200, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 1200, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[A_0]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
index a28673cf8e55..09b274de3021 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse-output.ll
@@ -22,16 +22,16 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RV64-NEXT:  [[ENTRY:.*]]:
 ; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
 ; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; RV64-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV64-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
@@ -79,16 +79,16 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV32-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RV32-NEXT:  [[ENTRY:.*]]:
 ; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
 ; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; RV32-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV32-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV32-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV32-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
@@ -138,16 +138,16 @@ define void @vector_reverse_i32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0:[0-9]+]] {
 ; RV64-UF2-NEXT:  [[ENTRY:.*]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
 ; RV64-UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
 ; RV64-UF2-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -228,16 +228,16 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
 ; RV64-NEXT:  [[ENTRY:.*]]:
 ; RV64-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; RV64-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; RV64-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64:       [[VECTOR_PH]]:
 ; RV64-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; RV64-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; RV64-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV64-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV64-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV64:       [[VECTOR_BODY]]:
@@ -285,16 +285,16 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV32-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
 ; RV32-NEXT:  [[ENTRY:.*]]:
 ; RV32-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; RV32-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; RV32-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV32-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV32:       [[VECTOR_PH]]:
 ; RV32-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; RV32-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; RV32-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV32-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV32-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV32-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV32-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV32-NEXT:    [[TMP6:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV32-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; RV32:       [[VECTOR_BODY]]:
@@ -344,16 +344,16 @@ define void @vector_reverse_f32(ptr noalias %A, ptr noalias %B) {
 ; RV64-UF2-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) #[[ATTR0]] {
 ; RV64-UF2-NEXT:  [[ENTRY:.*]]:
 ; RV64-UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; RV64-UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; RV64-UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1023, [[TMP1]]
 ; RV64-UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; RV64-UF2:       [[VECTOR_PH]]:
 ; RV64-UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; RV64-UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; RV64-UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1023, [[TMP3]]
 ; RV64-UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1023, [[N_MOD_VF]]
 ; RV64-UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; RV64-UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; RV64-UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
 ; RV64-UF2-NEXT:    [[TMP7:%.*]] = sub i64 1023, [[N_VEC]]
 ; RV64-UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
index ba4c4b6d58ad..41252f519155 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
@@ -214,7 +214,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.memcheck>:
 ; CHECK-NEXT:    IR %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %12 = mul i64 %11, 4
+; CHECK-NEXT:    IR %12 = mul nuw i64 %11, 4
 ; CHECK-NEXT:    IR %13 = mul i64 %12, 4
 ; CHECK-NEXT:    IR %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    IR %diff.check = icmp ult i64 %14, %13
@@ -222,11 +222,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.ph>:
 ; CHECK-NEXT:    IR %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %16 = mul i64 %15, 4
+; CHECK-NEXT:    IR %16 = mul nuw i64 %15, 4
 ; CHECK-NEXT:    IR %n.mod.vf = urem i64 %0, %16
 ; CHECK-NEXT:    IR %n.vec = sub i64 %0, %n.mod.vf
 ; CHECK-NEXT:    IR %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %18 = mul i64 %17, 4
+; CHECK-NEXT:    IR %18 = mul nuw i64 %17, 4
 ; CHECK-NEXT:    vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:    vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector.body
@@ -279,7 +279,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  for.body.preheader: ; preds = %entry
 ; CHECK-NEXT:    %0 = zext i32 %n to i64
 ; CHECK-NEXT:    %1 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %2 = mul i64 %1, 4
+; CHECK-NEXT:    %2 = mul nuw i64 %1, 4
 ; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
 ; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
 ; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
@@ -302,7 +302,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
 ; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %12 = mul i64 %11, 4
+; CHECK-NEXT:    %12 = mul nuw i64 %11, 4
 ; CHECK-NEXT:    %13 = mul i64 %12, 4
 ; CHECK-NEXT:    %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
@@ -312,11 +312,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
 ; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %16 = mul i64 %15, 4
+; CHECK-NEXT:    %16 = mul nuw i64 %15, 4
 ; CHECK-NEXT:    %n.mod.vf = urem i64 %0, %16
 ; CHECK-NEXT:    %n.vec = sub i64 %0, %n.mod.vf
 ; CHECK-NEXT:    %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %18 = mul i64 %17, 4
+; CHECK-NEXT:    %18 = mul nuw i64 %17, 4
 ; CHECK-NEXT:    %19 = sub i64 %0, %n.vec
 ; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
 ; CHECK-NEXT:    %20 = sub i32 %n, %.cast
@@ -623,7 +623,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.memcheck>:
 ; CHECK-NEXT:    IR %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %12 = mul i64 %11, 4
+; CHECK-NEXT:    IR %12 = mul nuw i64 %11, 4
 ; CHECK-NEXT:    IR %13 = mul i64 %12, 4
 ; CHECK-NEXT:    IR %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    IR %diff.check = icmp ult i64 %14, %13
@@ -631,11 +631,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  ir-bb<vector.ph>:
 ; CHECK-NEXT:    IR %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %16 = mul i64 %15, 4
+; CHECK-NEXT:    IR %16 = mul nuw i64 %15, 4
 ; CHECK-NEXT:    IR %n.mod.vf = urem i64 %0, %16
 ; CHECK-NEXT:    IR %n.vec = sub i64 %0, %n.mod.vf
 ; CHECK-NEXT:    IR %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    IR %18 = mul i64 %17, 4
+; CHECK-NEXT:    IR %18 = mul nuw i64 %17, 4
 ; CHECK-NEXT:    vp<%1> = DERIVED-IV ir<%0> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:    vp<%2> = DERIVED-IV ir<%n> + ir<%n.vec> * ir<-1>
 ; CHECK-NEXT:  Successor(s): vector.body
@@ -688,7 +688,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  for.body.preheader: ; preds = %entry
 ; CHECK-NEXT:    %0 = zext i32 %n to i64
 ; CHECK-NEXT:    %1 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %2 = mul i64 %1, 4
+; CHECK-NEXT:    %2 = mul nuw i64 %1, 4
 ; CHECK-NEXT:    %min.iters.check = icmp ult i64 %0, %2
 ; CHECK-NEXT:    br i1 %min.iters.check, label %scalar.ph, label %vector.scevcheck
 ; CHECK-NEXT:  LV: vectorizing VPBB:ir-bb<vector.scevcheck> in BB:vector.scevcheck
@@ -711,7 +711,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.memcheck: ; preds = %vector.scevcheck
 ; CHECK-NEXT:    %11 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %12 = mul i64 %11, 4
+; CHECK-NEXT:    %12 = mul nuw i64 %11, 4
 ; CHECK-NEXT:    %13 = mul i64 %12, 4
 ; CHECK-NEXT:    %14 = sub i64 %B1, %A2
 ; CHECK-NEXT:    %diff.check = icmp ult i64 %14, %13
@@ -721,11 +721,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
 ; CHECK-NEXT:  LV: filled BB:
 ; CHECK-NEXT:  vector.ph: ; preds = %vector.memcheck
 ; CHECK-NEXT:    %15 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %16 = mul i64 %15, 4
+; CHECK-NEXT:    %16 = mul nuw i64 %15, 4
 ; CHECK-NEXT:    %n.mod.vf = urem i64 %0, %16
 ; CHECK-NEXT:    %n.vec = sub i64 %0, %n.mod.vf
 ; CHECK-NEXT:    %17 = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    %18 = mul i64 %17, 4
+; CHECK-NEXT:    %18 = mul nuw i64 %17, 4
 ; CHECK-NEXT:    %19 = sub i64 %0, %n.vec
 ; CHECK-NEXT:    %.cast = trunc i64 %n.vec to i32
 ; CHECK-NEXT:    %20 = sub i32 %n, %.cast
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
index 3cba3de13b03..4fb02827b829 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll
@@ -11,16 +11,16 @@ define void @test(ptr %p) {
 ; CHECK-LABEL: @test(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -132,16 +132,16 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; CHECK-LABEL: @trivial_due_max_vscale(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -197,16 +197,16 @@ define void @no_high_lmul_or_interleave(ptr %p) {
 ; CHECK-LABEL: @no_high_lmul_or_interleave(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 200, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
index 5a658dbf0400..d1757c7e81ff 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll
@@ -16,16 +16,16 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; VLENUNK-LABEL: @vector_add(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; VLENUNK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -60,16 +60,16 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; VLEN128-LABEL: @vector_add(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; VLEN128-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; VLEN128-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLEN128-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -125,16 +125,16 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) {
 ; VLENUNK-LABEL: @vector_add_i32(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; VLENUNK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -169,16 +169,16 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) {
 ; VLEN128-LABEL: @vector_add_i32(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; VLEN128-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; VLEN128-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[V:%.*]], i64 0
 ; VLEN128-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -289,16 +289,16 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; VLENUNK-LABEL: @indexed_store(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; VLENUNK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -333,16 +333,16 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; VLEN128-LABEL: @indexed_store(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; VLEN128-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; VLEN128-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLEN128-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -395,16 +395,16 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; VLENUNK-LABEL: @indexed_load(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; VLENUNK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VLENUNK:       vector.body:
 ; VLENUNK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -444,16 +444,16 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; VLEN128-LABEL: @indexed_load(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; VLEN128-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VLEN128:       vector.body:
 ; VLEN128-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -513,16 +513,16 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; VLENUNK-LABEL: @splat_int(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; VLENUNK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -553,16 +553,16 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; VLEN128-LABEL: @splat_int(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; VLEN128-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; VLEN128-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; VLEN128-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -609,16 +609,16 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) {
 ; VLENUNK-LABEL: @splat_ptr(
 ; VLENUNK-NEXT:  entry:
 ; VLENUNK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLENUNK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLENUNK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLENUNK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLENUNK:       vector.ph:
 ; VLENUNK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLENUNK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLENUNK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLENUNK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLENUNK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; VLENUNK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; VLENUNK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; VLENUNK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[V:%.*]], i64 0
 ; VLENUNK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; VLENUNK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -649,16 +649,16 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) {
 ; VLEN128-LABEL: @splat_ptr(
 ; VLEN128-NEXT:  entry:
 ; VLEN128-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; VLEN128-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; VLEN128-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; VLEN128-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; VLEN128:       vector.ph:
 ; VLEN128-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; VLEN128-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; VLEN128-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; VLEN128-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; VLEN128-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; VLEN128-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; VLEN128-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; VLEN128-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[V:%.*]], i64 0
 ; VLEN128-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; VLEN128-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
index c6bcd7201777..ff9c58525e51 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll
@@ -12,13 +12,13 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -75,13 +75,13 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -136,13 +136,13 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -205,13 +205,13 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -260,13 +260,13 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -351,13 +351,13 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) {
 ; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP4]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
index 076ada825d0b..01b4ad2e66b8 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll
@@ -33,11 +33,11 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 {
 ; SCALABLE-LABEL: @select_icmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; SCALABLE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X:%.*]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -104,11 +104,11 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0
 ; SCALABLE-LABEL: @select_fcmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; SCALABLE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[X:%.*]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -173,11 +173,11 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 {
 ; SCALABLE-LABEL: @select_const_i32_from_icmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; SCALABLE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -240,11 +240,11 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64
 ; SCALABLE-LABEL: @select_i32_from_icmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; SCALABLE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -307,11 +307,11 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 {
 ; SCALABLE-LABEL: @select_const_i32_from_fcmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; SCALABLE-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -404,11 +404,11 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1
 ; SCALABLE-LABEL: @pred_select_const_i32_from_icmp
 ; SCALABLE:       vector.ph:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; SCALABLE-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; SCALABLE-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; SCALABLE:       vector.body:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
index 79590f5060ad..25dac366ef73 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll
@@ -7,18 +7,18 @@ define void @single_constant_stride_int_scaled(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_int_scaled(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[TMP5]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[TMP8:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP10:%.*]] = mul <vscale x 4 x i64> [[TMP8]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP10]]
@@ -78,16 +78,16 @@ define void @single_constant_stride_int_iv(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_int_iv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    [[IND_END:%.*]] = mul i64 [[N_VEC]], 64
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i64> [[TMP6]], splat (i64 64)
@@ -152,18 +152,18 @@ define void @single_constant_stride_ptr_iv(ptr %p) {
 ; CHECK-LABEL: @single_constant_stride_ptr_iv(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0
 ; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[TMP5]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-NEXT:    [[TMP18:%.*]] = mul i64 [[N_VEC]], 8
 ; CHECK-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[TMP18]]
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -171,7 +171,7 @@ define void @single_constant_stride_ptr_iv(ptr %p) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = mul i64 8, [[TMP8]]
 ; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP10]], 0
 ; CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP13]], i64 0
@@ -232,7 +232,7 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
 ; NOSTRIDED-LABEL: @single_stride_int_scaled(
 ; NOSTRIDED-NEXT:  entry:
 ; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
@@ -241,11 +241,11 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
 ; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
 ; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -314,7 +314,7 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
 ; NOSTRIDED-LABEL: @single_stride_int_iv(
 ; NOSTRIDED-NEXT:  entry:
 ; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
@@ -323,11 +323,11 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) {
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
 ; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
 ; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -438,7 +438,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    [[P3:%.*]] = ptrtoint ptr [[P:%.*]] to i64
 ; NOSTRIDED-NEXT:    [[P21:%.*]] = ptrtoint ptr [[P2:%.*]] to i64
 ; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 16, i64 [[TMP1]])
 ; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
@@ -447,18 +447,18 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_MEMCHECK:%.*]]
 ; NOSTRIDED:       vector.memcheck:
 ; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; NOSTRIDED-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
 ; NOSTRIDED-NEXT:    [[TMP6:%.*]] = sub i64 [[P21]], [[P3]]
 ; NOSTRIDED-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP6]], [[TMP5]]
 ; NOSTRIDED-NEXT:    br i1 [[DIFF_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
 ; NOSTRIDED-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; NOSTRIDED-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]]
 ; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; NOSTRIDED-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; NOSTRIDED-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -495,7 +495,7 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-LABEL: @double_stride_int_scaled(
 ; STRIDED-NEXT:  entry:
 ; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; STRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 80, i64 [[TMP1]])
 ; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -548,11 +548,11 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
 ; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
 ; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; STRIDED-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; STRIDED-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; STRIDED-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[STRIDE]], i64 0
 ; STRIDED-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
 ; STRIDED-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -618,7 +618,7 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-LABEL: @double_stride_int_iv(
 ; NOSTRIDED-NEXT:  entry:
 ; NOSTRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NOSTRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NOSTRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP1]])
 ; NOSTRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; NOSTRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_SCEVCHECK:%.*]]
@@ -627,11 +627,11 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) {
 ; NOSTRIDED-NEXT:    br i1 [[IDENT_CHECK]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NOSTRIDED:       vector.ph:
 ; NOSTRIDED-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; NOSTRIDED-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; NOSTRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]]
 ; NOSTRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; NOSTRIDED-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; NOSTRIDED-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; NOSTRIDED-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NOSTRIDED:       vector.body:
 ; NOSTRIDED-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -726,7 +726,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-LABEL: @double_stride_ptr_iv(
 ; STRIDED-NEXT:  entry:
 ; STRIDED-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; STRIDED-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; STRIDED-NEXT:    [[TMP2:%.*]] = call i64 @llvm.umax.i64(i64 32, i64 [[TMP1]])
 ; STRIDED-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP2]]
 ; STRIDED-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
@@ -750,11 +750,11 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; STRIDED:       vector.ph:
 ; STRIDED-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; STRIDED-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; STRIDED-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]]
 ; STRIDED-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; STRIDED-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; STRIDED-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; STRIDED-NEXT:    [[TMP10:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
 ; STRIDED-NEXT:    [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]]
 ; STRIDED-NEXT:    [[TMP11:%.*]] = mul i64 [[N_VEC]], [[STRIDE]]
@@ -765,7 +765,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[POINTER_PHI11:%.*]] = phi ptr [ [[P2]], [[VECTOR_PH]] ], [ [[PTR_IND12:%.*]], [[VECTOR_BODY]] ]
 ; STRIDED-NEXT:    [[TMP14:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP14]], 4
+; STRIDED-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP14]], 4
 ; STRIDED-NEXT:    [[TMP17:%.*]] = mul i64 [[STRIDE]], [[TMP13]]
 ; STRIDED-NEXT:    [[TMP18:%.*]] = mul i64 [[TMP15]], 0
 ; STRIDED-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP18]], i64 0
@@ -777,7 +777,7 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) {
 ; STRIDED-NEXT:    [[TMP21:%.*]] = mul <vscale x 4 x i64> [[TMP20]], [[DOTSPLAT10]]
 ; STRIDED-NEXT:    [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], <vscale x 4 x i64> [[TMP21]]
 ; STRIDED-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; STRIDED-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; STRIDED-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
 ; STRIDED-NEXT:    [[TMP25:%.*]] = mul i64 [[STRIDE]], [[TMP13]]
 ; STRIDED-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP23]], 0
 ; STRIDED-NEXT:    [[DOTSPLATINSERT13:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP26]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
index f83b5782d5ad..528cec077d8a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-cost.ll
@@ -9,16 +9,16 @@ define void @test_pr98413_zext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 97, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 97, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 97, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
@@ -82,16 +82,16 @@ define void @test_pr98413_sext_removed(ptr %src, ptr noalias %dst, i64 %x) {
 ; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]], i64 [[X:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 97, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 97, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 97, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x i64> poison, i64 [[X]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 8 x i64> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP6:%.*]] = trunc <vscale x 8 x i64> [[BROADCAST_SPLAT]] to <vscale x 8 x i8>
@@ -158,13 +158,13 @@ define void @truncate_to_i1_used_by_branch(i8 %x, ptr %dst) #0 {
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[TMP12:%.*]] = sub i32 [[TMP3]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 9, [[TMP12]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP10]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP10]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[DST]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -269,16 +269,16 @@ define void @icmp_only_first_op_truncated(ptr noalias %dst, i32 %x, i64 %N, i64
 ; CHECK-NEXT:    [[T:%.*]] = trunc i64 [[N]] to i32
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[V]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[N]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[T]], i64 0
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
index a94fcbffcf3b..72afff279e6b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/truncate-to-minimal-bitwidth-evl-crash.ll
@@ -12,13 +12,13 @@ define void @truncate_to_minimal_bitwidths_widen_cast_recipe(ptr %src) {
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 9, [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index 3a929f3e43a7..a256e92c823e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -12,7 +12,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[WIDE_TRIP_COUNT]], 1
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
@@ -25,13 +25,13 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 8
 ; CHECK-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP8]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 8
+; CHECK-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 8
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 8 x ptr> poison, ptr [[DSTV]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 8 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 8 x ptr> poison, <vscale x 8 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
index 827612cfe36d..71018451f59a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll
@@ -14,16 +14,16 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -94,13 +94,13 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-SCALABLE:       [[VECTOR_BODY]]:
 ; TF-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -185,16 +185,16 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -315,16 +315,16 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[B]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -425,13 +425,13 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x ptr> poison, ptr [[B]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 4 x ptr> poison, <vscale x 4 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[TMP5:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
@@ -551,16 +551,16 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; SCALABLE:       [[VECTOR_BODY]]:
 ; SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -631,13 +631,13 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; TF-SCALABLE:       [[VECTOR_BODY]]:
 ; TF-SCALABLE-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -722,16 +722,16 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -802,13 +802,13 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -893,16 +893,16 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -917,7 +917,7 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; SCALABLE-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 0
 ; SCALABLE-NEXT:    [[TMP11:%.*]] = add i64 [[INDEX]], 1
 ; SCALABLE-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; SCALABLE-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 2
+; SCALABLE-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 2
 ; SCALABLE-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
 ; SCALABLE-NEXT:    [[TMP15:%.*]] = extractelement <vscale x 2 x i64> [[TMP9]], i32 [[TMP14]]
 ; SCALABLE-NEXT:    store i64 [[TMP15]], ptr [[B]], align 8
@@ -989,13 +989,13 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x ptr> [[BROADCAST_SPLATINSERT]], <vscale x 2 x ptr> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
@@ -1117,16 +1117,16 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
@@ -1226,13 +1226,13 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT2:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT1:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT2]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 2 x ptr> poison, ptr [[B]], i64 0
@@ -1351,16 +1351,16 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ; SCALABLE-SAME: ptr noalias captures(none) [[A:%.*]], ptr noalias captures(none) [[B:%.*]], i64 [[V:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; SCALABLE-NEXT:  [[ENTRY:.*]]:
 ; SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; SCALABLE-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1025, [[TMP1]]
 ; SCALABLE-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; SCALABLE:       [[VECTOR_PH]]:
 ; SCALABLE-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; SCALABLE-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]]
 ; SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]]
 ; SCALABLE-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; SCALABLE-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; SCALABLE-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -1431,13 +1431,13 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap
 ; TF-SCALABLE-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; TF-SCALABLE:       [[VECTOR_PH]]:
 ; TF-SCALABLE-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; TF-SCALABLE-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; TF-SCALABLE-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; TF-SCALABLE-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; TF-SCALABLE-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; TF-SCALABLE-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; TF-SCALABLE-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; TF-SCALABLE-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i64> poison, i64 [[V]], i64 0
 ; TF-SCALABLE-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i64> poison, <vscale x 2 x i32> zeroinitializer
 ; TF-SCALABLE-NEXT:    br label %[[VECTOR_BODY:.*]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
index e1132e7b8935..16c575f5a817 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-bin-unary-ops-args.ll
@@ -19,19 +19,19 @@ define void @test_and(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -112,19 +112,19 @@ define void @test_or(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -205,19 +205,19 @@ define void @test_xor(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -298,19 +298,19 @@ define void @test_shl(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -391,19 +391,19 @@ define void @test_lshr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -484,19 +484,19 @@ define void @test_ashr(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -577,19 +577,19 @@ define void @test_add(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -670,19 +670,19 @@ define void @test_sub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -763,19 +763,19 @@ define void @test_mul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -856,19 +856,19 @@ define void @test_sdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -949,19 +949,19 @@ define void @test_udiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1042,19 +1042,19 @@ define void @test_srem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1135,19 +1135,19 @@ define void @test_urem(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP1]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 16
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 16
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP7]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 16
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 16
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1230,20 +1230,20 @@ define void @test_fadd(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1324,20 +1324,20 @@ define void @test_fsub(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1418,20 +1418,20 @@ define void @test_fmul(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1512,20 +1512,20 @@ define void @test_fdiv(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -1659,20 +1659,20 @@ define void @test_fneg(ptr nocapture %a, ptr nocapture readonly %b) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = sub i64 [[B1]], [[A2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 100, [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
index 3128e40144e3..325f3fd2b968 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
@@ -18,13 +18,13 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
@@ -35,13 +35,13 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -132,13 +132,13 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
@@ -149,13 +149,13 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -246,13 +246,13 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
@@ -263,13 +263,13 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -360,13 +360,13 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
@@ -377,13 +377,13 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[CONFLICT_RDX]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP28:%.*]] = mul i64 [[TMP27]], 4
+; IF-EVL-NEXT:    [[TMP28:%.*]] = mul nuw i64 [[TMP27]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP28]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP28]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -474,25 +474,25 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -573,25 +573,25 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP7]], [[TMP6]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; IF-EVL-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP9]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; IF-EVL-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -672,26 +672,26 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -780,26 +780,26 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP24:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP24]], [[TMP23]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; IF-EVL-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP26]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP26]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -888,26 +888,26 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP20:%.*]] = mul i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP21:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP21]], [[TMP20]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP22:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP23:%.*]] = mul i64 [[TMP22]], 4
+; IF-EVL-NEXT:    [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP23]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP23]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
index 50b600f8e8bd..107ca54c002f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -15,7 +15,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -30,13 +30,13 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -114,7 +114,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -129,13 +129,13 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -213,7 +213,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -228,13 +228,13 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -312,7 +312,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -327,13 +327,13 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -411,7 +411,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
@@ -426,13 +426,13 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP9]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 2
+; IF-EVL-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -512,26 +512,26 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -611,26 +611,26 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -710,26 +710,26 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -809,26 +809,26 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -908,26 +908,26 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
 ; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[A1]], [[B2]]
 ; IF-EVL-NEXT:    [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP8]], [[TMP7]]
 ; IF-EVL-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 2
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 2
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i64 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP11]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP10]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
index 3f17c95f7ca9..8faec471cf5a 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
@@ -27,18 +27,18 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
@@ -86,18 +86,18 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -142,16 +142,16 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -193,16 +193,16 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
 ; NO-VP-INLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -264,19 +264,19 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; IF-EVL-OUTLOOP-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT1]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -337,18 +337,18 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP7]], 1
 ; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP7]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -396,16 +396,16 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -451,16 +451,16 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
 ; NO-VP-INLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -571,16 +571,16 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
@@ -632,16 +632,16 @@ define i32 @step_cond_add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
 ; NO-VP-INLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-INLOOP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
 ; NO-VP-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
@@ -761,16 +761,16 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[N]], [[TMP2]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP7]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP11:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-OUTLOOP-NEXT:    [[TMP14:%.*]] = mul <vscale x 4 x i32> [[TMP12]], splat (i32 1)
@@ -826,16 +826,16 @@ define i32 @step_cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
 ; NO-VP-INLOOP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
 ; NO-VP-INLOOP-NEXT:    [[TMP8:%.*]] = mul <vscale x 4 x i32> [[TMP6]], splat (i32 1)
 ; NO-VP-INLOOP-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i32> zeroinitializer, [[TMP8]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-div.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-div.ll
index 2cada3a9ec16..3e83d8a757b5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-div.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-div.ll
@@ -16,13 +16,13 @@ define void @test_sdiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -112,13 +112,13 @@ define void @test_udiv(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -207,13 +207,13 @@ define void @test_srem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -302,13 +302,13 @@ define void @test_urem(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
 ; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; IF-EVL:       [[VECTOR_BODY]]:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
index a138fbf5b6e9..f651f2295b3b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
@@ -17,21 +17,21 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP25:%.*]] = trunc i64 [[TMP8]] to i32
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP11]]
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -79,18 +79,18 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 4
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP8]]
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -110,7 +110,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
 ; NO-VP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 4
+; NO-VP-NEXT:    [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 4
 ; NO-VP-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP19]]
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
@@ -158,25 +158,25 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP32:%.*]] = trunc i64 [[TMP8]] to i32
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 22, i32 [[TMP14]]
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -228,22 +228,22 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 4
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP8]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; NO-VP-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 22, i32 [[TMP11]]
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -265,11 +265,11 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    br i1 [[TMP20]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
 ; NO-VP-NEXT:    [[TMP21:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP22:%.*]] = mul i32 [[TMP21]], 4
+; NO-VP-NEXT:    [[TMP22:%.*]] = mul nuw i32 [[TMP21]], 4
 ; NO-VP-NEXT:    [[TMP23:%.*]] = sub i32 [[TMP22]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP23]]
 ; NO-VP-NEXT:    [[TMP24:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP25:%.*]] = mul i32 [[TMP24]], 4
+; NO-VP-NEXT:    [[TMP25:%.*]] = mul nuw i32 [[TMP24]], 4
 ; NO-VP-NEXT:    [[TMP26:%.*]] = sub i32 [[TMP25]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT3:%.*]] = extractelement <vscale x 4 x i32> [[TMP15]], i32 [[TMP26]]
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
@@ -320,29 +320,29 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP39:%.*]] = trunc i64 [[TMP8]] to i32
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; IF-EVL-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP11]]
 ; IF-EVL-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 4
+; IF-EVL-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
 ; IF-EVL-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 22, i32 [[TMP14]]
 ; IF-EVL-NEXT:    [[TMP15:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP16:%.*]] = mul i32 [[TMP15]], 4
+; IF-EVL-NEXT:    [[TMP16:%.*]] = mul nuw i32 [[TMP15]], 4
 ; IF-EVL-NEXT:    [[TMP17:%.*]] = sub i32 [[TMP16]], 1
 ; IF-EVL-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 11, i32 [[TMP17]]
 ; IF-EVL-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -400,26 +400,26 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 4
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP8]]
 ; NO-VP-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; NO-VP-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT1:%.*]] = insertelement <vscale x 4 x i32> poison, i32 22, i32 [[TMP11]]
 ; NO-VP-NEXT:    [[TMP12:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP13:%.*]] = mul i32 [[TMP12]], 4
+; NO-VP-NEXT:    [[TMP13:%.*]] = mul nuw i32 [[TMP12]], 4
 ; NO-VP-NEXT:    [[TMP14:%.*]] = sub i32 [[TMP13]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT3:%.*]] = insertelement <vscale x 4 x i32> poison, i32 11, i32 [[TMP14]]
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -444,15 +444,15 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    br i1 [[TMP25]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
 ; NO-VP-NEXT:    [[TMP26:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP27:%.*]] = mul i32 [[TMP26]], 4
+; NO-VP-NEXT:    [[TMP27:%.*]] = mul nuw i32 [[TMP26]], 4
 ; NO-VP-NEXT:    [[TMP28:%.*]] = sub i32 [[TMP27]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP28]]
 ; NO-VP-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 4
+; NO-VP-NEXT:    [[TMP30:%.*]] = mul nuw i32 [[TMP29]], 4
 ; NO-VP-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT5:%.*]] = extractelement <vscale x 4 x i32> [[TMP18]], i32 [[TMP31]]
 ; NO-VP-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 4
+; NO-VP-NEXT:    [[TMP33:%.*]] = mul nuw i32 [[TMP32]], 4
 ; NO-VP-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP33]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT6:%.*]] = extractelement <vscale x 4 x i32> [[TMP19]], i32 [[TMP34]]
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
@@ -526,18 +526,18 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; NO-VP-NEXT:  [[ENTRY:.*]]:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TC]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; NO-VP:       [[VECTOR_PH]]:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TC]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[TC]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP7:%.*]] = mul i32 [[TMP6]], 4
+; NO-VP-NEXT:    [[TMP7:%.*]] = mul nuw i32 [[TMP6]], 4
 ; NO-VP-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP7]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 33, i32 [[TMP8]]
 ; NO-VP-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -557,11 +557,11 @@ define i32 @FOR_reduction(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; NO-VP-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; NO-VP:       [[MIDDLE_BLOCK]]:
 ; NO-VP-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 4
+; NO-VP-NEXT:    [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 4
 ; NO-VP-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 2
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP19]]
 ; NO-VP-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 4
+; NO-VP-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP20]], 4
 ; NO-VP-NEXT:    [[TMP22:%.*]] = sub i32 [[TMP21]], 1
 ; NO-VP-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP22]]
 ; NO-VP-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TC]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
index d9b1981869b2..c8d52efde4ac 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
@@ -16,18 +16,18 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -67,16 +67,16 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @add(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -241,18 +241,18 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -292,16 +292,16 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @or(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -356,18 +356,18 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -407,16 +407,16 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @and(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -471,18 +471,18 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -522,16 +522,16 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @xor(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -586,18 +586,18 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -638,16 +638,16 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -704,18 +704,18 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -756,16 +756,16 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -822,18 +822,18 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -874,16 +874,16 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -940,18 +940,18 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -992,16 +992,16 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1058,18 +1058,18 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1109,16 +1109,16 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-LABEL: @fadd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1283,18 +1283,18 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1336,16 +1336,16 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1403,18 +1403,18 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1456,16 +1456,16 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1747,18 +1747,18 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1804,16 +1804,16 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-LABEL: @fmuladd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1876,18 +1876,18 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1931,16 +1931,16 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_icmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2000,18 +2000,18 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2055,16 +2055,16 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_fcmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
index cd246053bcb3..38ef88457c6c 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
@@ -15,19 +15,19 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
@@ -56,7 +56,7 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; IF-EVL-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; IF-EVL-NEXT:    [[TMP29:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i32 0
 ; IF-EVL-NEXT:    [[TMP30:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP31:%.*]] = mul i64 [[TMP30]], 4
+; IF-EVL-NEXT:    [[TMP31:%.*]] = mul nuw i64 [[TMP30]], 4
 ; IF-EVL-NEXT:    [[TMP32:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i64 [[TMP31]]
 ; IF-EVL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP25]], ptr [[TMP29]], i32 4, <vscale x 4 x i1> [[TMP19]])
 ; IF-EVL-NEXT:    call void @llvm.masked.store.nxv4i32.p0(<vscale x 4 x i32> [[TMP26]], ptr [[TMP32]], i32 4, <vscale x 4 x i1> [[TMP20]])
@@ -87,16 +87,16 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; NO-VP-LABEL: @interleave(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP8]], 2
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -119,7 +119,7 @@ define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; NO-VP-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]]
 ; NO-VP-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i32 0
 ; NO-VP-NEXT:    [[TMP25:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP26:%.*]] = mul i64 [[TMP25]], 4
+; NO-VP-NEXT:    [[TMP26:%.*]] = mul nuw i64 [[TMP25]], 4
 ; NO-VP-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP22]], i64 [[TMP26]]
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP20]], ptr [[TMP24]], align 4
 ; NO-VP-NEXT:    store <vscale x 4 x i32> [[TMP21]], ptr [[TMP27]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
index 3e7483143c88..6d54b219a346 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
@@ -27,7 +27,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-NEXT:  entry:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; IF-EVL-OUTLOOP:       vector.memcheck:
@@ -40,13 +40,13 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[ENTRY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP7]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; IF-EVL-OUTLOOP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP10:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; IF-EVL-OUTLOOP-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL-OUTLOOP:       vector.body:
@@ -91,7 +91,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-NEXT:  entry:
 ; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-INLOOP-NEXT:    br i1 [[TMP4]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; IF-EVL-INLOOP:       vector.memcheck:
@@ -104,13 +104,13 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP10:%.*]] = sub i64 [[TMP9]], 1
 ; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP10]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP9]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; IF-EVL-INLOOP-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; IF-EVL-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL-INLOOP:       vector.body:
 ; IF-EVL-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -152,7 +152,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-OUTLOOP-NEXT:  entry:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; NO-VP-OUTLOOP:       vector.memcheck:
@@ -165,11 +165,11 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-OUTLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NO-VP-OUTLOOP:       vector.ph:
 ; NO-VP-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; NO-VP-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-OUTLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-OUTLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-OUTLOOP-NEXT:    [[TMP8:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START]], i32 0
 ; NO-VP-OUTLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-OUTLOOP:       vector.body:
@@ -208,7 +208,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; NO-VP-INLOOP-NEXT:  entry:
 ; NO-VP-INLOOP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-INLOOP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; NO-VP-INLOOP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; NO-VP-INLOOP:       vector.memcheck:
@@ -221,11 +221,11 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; NO-VP-INLOOP-NEXT:    br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
 ; NO-VP-INLOOP:       vector.ph:
 ; NO-VP-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP5]]
 ; NO-VP-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-INLOOP-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; NO-VP-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; NO-VP-INLOOP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP-INLOOP:       vector.body:
 ; NO-VP-INLOOP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
index 64f2bab302b8..a05aabc063b1 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
@@ -14,18 +14,18 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP19:%.*]] = sub i32 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i32 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP19]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label [[FOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[IV:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ]
@@ -62,16 +62,16 @@ define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; NO-VP-LABEL: @iv32(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], [[TMP10]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP11:%.*]] = mul i32 [[TMP1]], 4
+; NO-VP-NEXT:    [[TMP11:%.*]] = mul nuw i32 [[TMP1]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP11]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; NO-VP-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP2]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
index 723ee6402672..dcd15087ce3d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
@@ -17,18 +17,18 @@ define void @trip_count_max_1024(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TC]], i64 1)
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP6]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -92,18 +92,18 @@ define void @overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK:       [[LOOP_PREHEADER]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 2
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC]], [[TMP6]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 2
+; CHECK-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -168,13 +168,13 @@ define void @no_overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TC_ADD]], [[TMP2]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
index aad20331e29f..7609182c7690 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
@@ -14,18 +14,18 @@ define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
index 24983c21a8ac..bc28918dac68 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
@@ -16,18 +16,18 @@ define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index af36f184ea82..53e0b2f45aaa 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -14,18 +14,18 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -67,16 +67,16 @@ define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @add(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -243,18 +243,18 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -296,16 +296,16 @@ define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @or(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -361,18 +361,18 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> splat (i32 -1), i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -414,16 +414,16 @@ define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @and(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> splat (i32 -1), i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -479,18 +479,18 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -532,16 +532,16 @@ define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @xor(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x i32> zeroinitializer, i32 [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -597,18 +597,18 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -653,16 +653,16 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -722,18 +722,18 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -778,16 +778,16 @@ define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @smax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -847,18 +847,18 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -903,16 +903,16 @@ define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -972,18 +972,18 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1028,16 +1028,16 @@ define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; NO-VP-LABEL: @umax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1097,18 +1097,18 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -1150,16 +1150,16 @@ define float @fadd(ptr %a, i64 %n, float %start) {
 ; NO-VP-LABEL: @fadd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -1326,18 +1326,18 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1382,16 +1382,16 @@ define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmin(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1451,18 +1451,18 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; IF-EVL-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1507,16 +1507,16 @@ define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; NO-VP-LABEL: @fmax(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x float> poison, float [[START:%.*]], i64 0
 ; NO-VP-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x float> [[BROADCAST_SPLATINSERT]], <vscale x 4 x float> poison, <vscale x 4 x i32> zeroinitializer
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -1800,18 +1800,18 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
@@ -1858,16 +1858,16 @@ define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; NO-VP-LABEL: @fmuladd(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    [[TMP6:%.*]] = insertelement <vscale x 4 x float> splat (float -0.000000e+00), float [[START:%.*]], i32 0
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
@@ -1930,18 +1930,18 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -1985,16 +1985,16 @@ define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_icmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2054,18 +2054,18 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; IF-EVL-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -2109,16 +2109,16 @@ define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; NO-VP-LABEL: @anyof_fcmp(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
index 4d8166eaa46f..96db5bf4e9ac 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reverse-load-store.ll
@@ -15,13 +15,13 @@ define void @reverse_load_store(i64 %startval, ptr noalias %ptr, ptr noalias %pt
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -113,13 +113,13 @@ define void @reverse_load_store_masked(i64 %startval, ptr noalias %ptr, ptr noal
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1024, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -244,13 +244,13 @@ define void @multiple_reverse_vector_pointer(ptr noalias %a, ptr noalias %b, ptr
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 16
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 16
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 1025, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 16
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 16
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
index ee32127f2889..336c242c13ad 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-safe-dep-distance.ll
@@ -18,13 +18,13 @@ define void @test(ptr %p) {
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 200, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -333,13 +333,13 @@ define void @trivial_due_max_vscale(ptr %p) {
 ; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; IF-EVL-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; IF-EVL-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 200, [[TMP2]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 2
+; IF-EVL-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 2
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index 82e8d3d6c611..81f52627b179 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -15,18 +15,18 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 3, [[SPEC_SELECT]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = sub i64 [[TMP6]], 1
 ; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], [[TMP7]]
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP6]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 2
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 2
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
index 579bc450b83d..f8f397212e0e 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
@@ -14,18 +14,18 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-NEXT:  entry:
 ; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
 ; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
 ; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP10:%.*]] = mul i64 [[TMP9]], 4
+; IF-EVL-NEXT:    [[TMP10:%.*]] = mul nuw i64 [[TMP9]], 4
 ; IF-EVL-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; IF-EVL:       vector.body:
 ; IF-EVL-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -70,16 +70,16 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-LABEL: @foo(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; NO-VP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 8a3c334db187..69d4f96f7ba2 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -21,13 +21,13 @@
 
  ; IF-EVL:      ir-bb<vector.ph>:
  ; IF-EVL-NEXT:   IR   %4 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %5 = mul i64 %4, 4
+ ; IF-EVL-NEXT:   IR   %5 = mul nuw i64 %4, 4
  ; IF-EVL-NEXT:   IR   %6 = sub i64 %5, 1
  ; IF-EVL-NEXT:   IR   %n.rnd.up = add i64 %N, %6
  ; IF-EVL-NEXT:   IR   %n.mod.vf = urem i64 %n.rnd.up, %5
  ; IF-EVL-NEXT:   IR   %n.vec = sub i64 %n.rnd.up, %n.mod.vf
  ; IF-EVL-NEXT:   IR   %7 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %8 = mul i64 %7, 4
+ ; IF-EVL-NEXT:   IR   %8 = mul nuw i64 %7, 4
  ; IF-EVL-NEXT: Successor(s): vector.body
  ; IF-EVL-EMPTY:
  ; IF-EVL-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
index 2135fc636791..4fda9d34b9dc 100644
--- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
+++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll
@@ -16,16 +16,16 @@ define void @foo() {
 ; CHECK-LABEL: define void @foo() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP18:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP19:%.*]] = mul i64 [[TMP18]], 4
+; CHECK-NEXT:    [[TMP19:%.*]] = mul nuw i64 [[TMP18]], 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul <vscale x 4 x i64> [[TMP4]], splat (i64 1)
 ; CHECK-NEXT:    [[INDUCTION:%.*]] = add <vscale x 4 x i64> zeroinitializer, [[TMP6]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
index bb84dbf8ed23..f429677e3875 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll
@@ -20,12 +20,12 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = add i64 [[B1]], -4
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
@@ -33,13 +33,13 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP16]]
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -60,11 +60,11 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP24]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP25:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP26:%.*]] = mul i32 [[TMP25]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP26:%.*]] = mul nuw i32 [[TMP25]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP27:%.*]] = sub i32 [[TMP26]], 2
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP27]]
 ; CHECK-VF4UF1-NEXT:    [[TMP28:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP29:%.*]] = mul i32 [[TMP28]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP29:%.*]] = mul nuw i32 [[TMP28]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP30:%.*]] = sub i32 [[TMP29]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP30]]
 ; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
@@ -87,12 +87,12 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = zext i32 [[TMP0]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = add i64 [[B1]], -4
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = sub i64 [[TMP8]], [[A2]]
@@ -100,13 +100,13 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP11]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[PRE_LOAD]], i32 [[TMP16]]
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -117,7 +117,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP17]]
 ; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = mul i64 [[TMP20]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = mul nuw i64 [[TMP20]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 [[TMP21]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP19]], align 4
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i32>, ptr [[TMP22]], align 4
@@ -128,7 +128,7 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = add <vscale x 4 x i32> [[WIDE_LOAD3]], [[TMP24]]
 ; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul i64 [[TMP29]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul nuw i64 [[TMP29]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP30]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP26]], ptr [[TMP28]], align 4
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP27]], ptr [[TMP31]], align 4
@@ -137,11 +137,11 @@ define i32 @recurrence_1(ptr nocapture readonly %a, ptr nocapture %b, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP32]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = mul i32 [[TMP33]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = mul nuw i32 [[TMP33]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP35:%.*]] = sub i32 [[TMP34]], 2
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD3]], i32 [[TMP35]]
 ; CHECK-VF4UF2-NEXT:    [[TMP36:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul i32 [[TMP36]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul nuw i32 [[TMP36]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP38:%.*]] = sub i32 [[TMP37]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD3]], i32 [[TMP38]]
 ; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
@@ -194,18 +194,18 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP9]]
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -228,7 +228,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP19:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[TMP17]])
 ; CHECK-VF4UF1-NEXT:    [[TMP20:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = mul i32 [[TMP20]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP21:%.*]] = mul nuw i32 [[TMP20]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = sub i32 [[TMP21]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD]], i32 [[TMP22]]
 ; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -256,18 +256,18 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i32, ptr [[ARRAYIDX2_PHI_TRANS_INSERT]], align 4
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = zext i32 [[N]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul i32 [[TMP7]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul nuw i32 [[TMP7]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = sub i32 [[TMP8]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[DOTPRE]], i32 [[TMP9]]
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -279,7 +279,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i32>, ptr [[TMP11]], align 4
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD2]] = load <vscale x 4 x i32>, ptr [[TMP14]], align 4
@@ -302,7 +302,7 @@ define i32 @recurrence_2(ptr nocapture readonly %a, i32 %n) {
 ; CHECK-VF4UF2-NEXT:    [[RDX_MINMAX:%.*]] = call <vscale x 4 x i32> @llvm.smin.nxv4i32(<vscale x 4 x i32> [[TMP25]], <vscale x 4 x i32> [[TMP26]])
 ; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = call i32 @llvm.vector.reduce.smin.nxv4i32(<vscale x 4 x i32> [[RDX_MINMAX]])
 ; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = mul nuw i32 [[TMP29]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[WIDE_LOAD2]], i32 [[TMP31]]
 ; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]]
@@ -371,7 +371,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
@@ -391,16 +391,16 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP17:%.*]] = mul nuw i32 [[TMP16]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP17]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[TMP0]], i32 [[TMP18]]
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -424,7 +424,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP29:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP30:%.*]] = mul i32 [[TMP29]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP30:%.*]] = mul nuw i32 [[TMP29]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP31:%.*]] = sub i32 [[TMP30]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP31]]
 ; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
@@ -451,7 +451,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = add i32 [[N]], -1
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = zext i32 [[TMP1]] to i64
 ; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul i64 [[TMP3]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], [[TMP4]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
@@ -471,16 +471,16 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul i64 [[TMP11]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP12]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP13]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP13]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = add i64 1, [[N_VEC]]
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x double> poison, double [[CONV1]], i64 0
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x double> [[BROADCAST_SPLATINSERT]], <vscale x 4 x double> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = mul i32 [[TMP16]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = mul nuw i32 [[TMP16]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP18:%.*]] = sub i32 [[TMP17]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[TMP0]], i32 [[TMP18]]
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -491,7 +491,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[OFFSET_IDX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = mul i64 [[TMP21]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP22:%.*]] = mul nuw i64 [[TMP21]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP23:%.*]] = getelementptr inbounds i16, ptr [[TMP19]], i64 [[TMP22]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP20]], align 2, !alias.scope [[META6:![0-9]+]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD4]] = load <vscale x 4 x i16>, ptr [[TMP23]], align 2, !alias.scope [[META6]]
@@ -508,7 +508,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = getelementptr inbounds double, ptr [[B]], i64 [[OFFSET_IDX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP35:%.*]] = getelementptr inbounds double, ptr [[TMP34]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP36:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul i64 [[TMP36]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP37:%.*]] = mul nuw i64 [[TMP36]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP38:%.*]] = getelementptr inbounds double, ptr [[TMP34]], i64 [[TMP37]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP32]], ptr [[TMP35]], align 8, !alias.scope [[META9:![0-9]+]], !noalias [[META6]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x double> [[TMP33]], ptr [[TMP38]], align 8, !alias.scope [[META9]], !noalias [[META6]]
@@ -517,7 +517,7 @@ define void @recurrence_3(ptr nocapture readonly %a, ptr nocapture %b, i32 %n, f
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP39]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP40:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP41:%.*]] = mul i32 [[TMP40]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP41:%.*]] = mul nuw i32 [[TMP40]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP42:%.*]] = sub i32 [[TMP41]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD4]], i32 [[TMP42]]
 ; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]]
@@ -570,16 +570,16 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF1-LABEL: define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -598,16 +598,16 @@ define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-LABEL: define i64 @constant_folded_previous_value() {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1000, [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1000, [[TMP3]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 1000, [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_BODY]]:
 ; CHECK-VF4UF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
@@ -648,16 +648,16 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF1-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; CHECK-VF4UF1-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
@@ -677,11 +677,11 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 2
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP13]]
 ; CHECK-VF4UF1-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP16]]
 ; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i32 96, [[N_VEC]]
@@ -696,16 +696,16 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF2-SAME: ptr [[CVAL:%.*]], i32 [[X:%.*]]) {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 96, [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i32 96, [[TMP3]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i32 96, [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], 2
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[X]], i64 0
 ; CHECK-VF4UF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
@@ -726,11 +726,11 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x)  {
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul i32 [[TMP11]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP12:%.*]] = mul nuw i32 [[TMP11]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = sub i32 [[TMP12]], 2
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT_FOR_PHI:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP13]]
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul i32 [[TMP14]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = mul nuw i32 [[TMP14]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = sub i32 [[TMP15]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i32> [[TMP9]], i32 [[TMP16]]
 ; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i32 96, [[N_VEC]]
@@ -769,7 +769,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF1-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
 ; CHECK-VF4UF1-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-VF4UF1-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-VF4UF1-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_MEMCHECK]]:
@@ -785,13 +785,13 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF1:       [[VECTOR_PH]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
 ; CHECK-VF4UF1-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-VF4UF1-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-VF4UF1-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP11]]
 ; CHECK-VF4UF1-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -814,7 +814,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF1-NEXT:    br i1 [[TMP21]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-VF4UF1:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF1-NEXT:    [[TMP22:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF1-NEXT:    [[TMP23:%.*]] = mul i32 [[TMP22]], 4
+; CHECK-VF4UF1-NEXT:    [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 4
 ; CHECK-VF4UF1-NEXT:    [[TMP24:%.*]] = sub i32 [[TMP23]], 1
 ; CHECK-VF4UF1-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD]], i32 [[TMP24]]
 ; CHECK-VF4UF1-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
@@ -830,7 +830,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:  [[ENTRY:.*]]:
 ; CHECK-VF4UF2-NEXT:    [[DOTPRE:%.*]] = load i16, ptr [[A]], align 2
 ; CHECK-VF4UF2-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 8
 ; CHECK-VF4UF2-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], [[TMP1]]
 ; CHECK-VF4UF2-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_MEMCHECK]]:
@@ -846,13 +846,13 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]]
 ; CHECK-VF4UF2:       [[VECTOR_PH]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul i64 [[TMP5]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 8
 ; CHECK-VF4UF2-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]]
 ; CHECK-VF4UF2-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-VF4UF2-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP7]], 8
+; CHECK-VF4UF2-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP7]], 8
 ; CHECK-VF4UF2-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = mul i32 [[TMP9]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP10:%.*]] = mul nuw i32 [[TMP9]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP11:%.*]] = sub i32 [[TMP10]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_INIT:%.*]] = insertelement <vscale x 4 x i16> poison, i16 [[DOTPRE]], i32 [[TMP11]]
 ; CHECK-VF4UF2-NEXT:    br label %[[VECTOR_BODY:.*]]
@@ -863,7 +863,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[TMP12]]
 ; CHECK-VF4UF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP15:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = mul i64 [[TMP15]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP16:%.*]] = mul nuw i64 [[TMP15]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i16, ptr [[TMP13]], i64 [[TMP16]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 4 x i16>, ptr [[TMP14]], align 2, !alias.scope [[META17:![0-9]+]]
 ; CHECK-VF4UF2-NEXT:    [[WIDE_LOAD3]] = load <vscale x 4 x i16>, ptr [[TMP17]], align 2, !alias.scope [[META17]]
@@ -878,7 +878,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]]
 ; CHECK-VF4UF2-NEXT:    [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i32 0
 ; CHECK-VF4UF2-NEXT:    [[TMP28:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = mul i64 [[TMP28]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP29:%.*]] = mul nuw i64 [[TMP28]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP26]], i64 [[TMP29]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP24]], ptr [[TMP27]], align 4, !alias.scope [[META20:![0-9]+]], !noalias [[META17]]
 ; CHECK-VF4UF2-NEXT:    store <vscale x 4 x i32> [[TMP25]], ptr [[TMP30]], align 4, !alias.scope [[META20]], !noalias [[META17]]
@@ -887,7 +887,7 @@ define void @sink_after(ptr %a, ptr %b, i64 %n) {
 ; CHECK-VF4UF2-NEXT:    br i1 [[TMP31]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; CHECK-VF4UF2:       [[MIDDLE_BLOCK]]:
 ; CHECK-VF4UF2-NEXT:    [[TMP32:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = mul i32 [[TMP32]], 4
+; CHECK-VF4UF2-NEXT:    [[TMP33:%.*]] = mul nuw i32 [[TMP32]], 4
 ; CHECK-VF4UF2-NEXT:    [[TMP34:%.*]] = sub i32 [[TMP33]], 1
 ; CHECK-VF4UF2-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <vscale x 4 x i16> [[WIDE_LOAD3]], i32 [[TMP34]]
 ; CHECK-VF4UF2-NEXT:    [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
index 10f96284c018..62649412deb8 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll
@@ -11,16 +11,16 @@ define void @add_ind64_unrolled(ptr noalias nocapture %a, ptr noalias nocapture
 ; CHECK-LABEL: @add_ind64_unrolled(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP8:%.*]] = shl i64 [[TMP4]], 1
+; CHECK-NEXT:    [[TMP8:%.*]] = shl nuw i64 [[TMP4]], 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 2 x i64> @llvm.stepvector.nxv2i64()
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
@@ -92,12 +92,12 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no
 ; CHECK-LABEL: @add_ind64_unrolled_nxv1i64(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 1
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 1
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 1
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 1
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
@@ -177,16 +177,16 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @add_unique_ind32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = shl i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32
 ; CHECK-NEXT:    [[IND_END:%.*]] = shl i32 [[DOTCAST]], 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = call <vscale x 4 x i32> @llvm.stepvector.nxv4i32()
@@ -254,16 +254,16 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) {
 ; CHECK-LABEL: @add_unique_indf32(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = shl i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = shl i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP6:%.*]] = shl i64 [[TMP5]], 2
+; CHECK-NEXT:    [[TMP6:%.*]] = shl nuw i64 [[TMP5]], 2
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float
 ; CHECK-NEXT:    [[TMP4:%.*]] = fmul float [[DOTCAST]], 2.000000e+00
 ; CHECK-NEXT:    [[IND_END:%.*]] = fadd float [[TMP4]], 0.000000e+00
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll b/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
index 15db687ba64f..b0029a4e0d06 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-iv-outside-user.ll
@@ -8,16 +8,16 @@ define i32 @iv_live_out_wide(ptr %dst) {
 ; CHECK-NEXT:    [[STEP_1:%.*]] = sext i8 0 to i32
 ; CHECK-NEXT:    [[STEP_2:%.*]] = add nsw i32 [[STEP_1]], 1
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i32 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 2000, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 2000, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 2000, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 2
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul i32 [[TMP5]], 2
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 2 x i32> poison, i32 [[STEP_2]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 2 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 2 x i32> poison, <vscale x 2 x i32> zeroinitializer
@@ -34,7 +34,7 @@ define i32 @iv_live_out_wide(ptr %dst) {
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[DST]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i32 0
 ; CHECK-NEXT:    [[TMP12:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP13:%.*]] = mul i64 [[TMP12]], 2
+; CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i64 [[TMP12]], 2
 ; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i16, ptr [[TMP10]], i64 [[TMP13]]
 ; CHECK-NEXT:    store <vscale x 2 x i16> zeroinitializer, ptr [[TMP11]], align 2
 ; CHECK-NEXT:    store <vscale x 2 x i16> zeroinitializer, ptr [[TMP14]], align 2
@@ -45,7 +45,7 @@ define i32 @iv_live_out_wide(ptr %dst) {
 ; CHECK-NEXT:    br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
 ; CHECK-NEXT:    [[TMP17:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP18:%.*]] = mul i32 [[TMP17]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = mul nuw i32 [[TMP17]], 2
 ; CHECK-NEXT:    [[TMP19:%.*]] = sub i32 [[TMP18]], 1
 ; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <vscale x 2 x i32> [[TMP15]], i32 [[TMP19]]
 ; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 2000, [[N_VEC]]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
index e1c14f9f6b71..4a1d7a2376dd 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll
@@ -12,16 +12,16 @@ define void @test(ptr %d) {
 ; CHECK-NEXT:    [[ARR:%.*]] = alloca [1024 x i32], align 16
 ; CHECK-NEXT:    call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]])
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -83,16 +83,16 @@ define void @testloopvariant(ptr %d) {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[ARR:%.*]] = alloca [1024 x i32], align 16
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 2
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 2
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 128, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 2
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 2
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
index 3f9ca05688de..f384d3c15ca6 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll
@@ -4,16 +4,16 @@
 ; CHECKUF1: for.body.preheader:
 ; CHECKUF1-DAG: %wide.trip.count = zext nneg i32 %N to i64
 ; CHECKUF1-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
 ; CHECKUF1-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX4]], %wide.trip.count
 
 ; CHECKUF1: vector.ph:
 ; CHECKUF1-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1-DAG:  %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1-DAG:  %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
 ; CHECKUF1-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]]
 ; CHECKUF1:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
 ; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2
+; CHECKUF1: %[[VSCALEX4:.*]] = shl nuw i64 %[[VSCALE]], 2
 
 ; CHECKUF1: vector.body:
 ; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
@@ -33,16 +33,16 @@
 ; CHECKUF2: for.body.preheader:
 ; CHECKUF2-DAG: %wide.trip.count = zext nneg i32 %N to i64
 ; CHECKUF2-DAG: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
 ; CHECKUF2-DAG: %min.iters.check = icmp ugt i64 %[[VSCALEX8]], %wide.trip.count
 
 ; CHECKUF2: vector.ph:
 ; CHECKUF2-DAG:  %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2-DAG:  %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2-DAG:  %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
 ; CHECKUF2-DAG:  %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]]
 ; CHECKUF2:      %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf
 ; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64()
-; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3
+; CHECKUF2: %[[VSCALEX8:.*]] = shl nuw i64 %[[VSCALE]], 3
 
 ; CHECKUF2: vector.body:
 ; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
index 04b4c6fe58db..079f6b73e886 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll
@@ -7,16 +7,16 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-LABEL: @reduction_add_trunc(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP30:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP31:%.*]] = mul i32 [[TMP30]], 16
+; CHECK-NEXT:    [[TMP31:%.*]] = mul nuw i32 [[TMP30]], 16
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 256, [[TMP31]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i32 [[TMP2]], 16
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i32 [[TMP2]], 16
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 256, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 256, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i32 [[TMP4]], 16
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 16
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
@@ -27,7 +27,7 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) {
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i32 [[INDEX]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0
 ; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP11:%.*]] = mul i64 [[TMP10]], 8
+; CHECK-NEXT:    [[TMP11:%.*]] = mul nuw i64 [[TMP10]], 8
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i64 [[TMP11]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <vscale x 8 x i8>, ptr [[TMP9]], align 4
 ; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <vscale x 8 x i8>, ptr [[TMP12]], align 4
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
index af83c13bdfdc..4b8ff8677468 100644
--- a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
+++ b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll
@@ -5,16 +5,16 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64
 ; CHECK-LABEL: @trunc_minimal_bitwidth(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP7:%.*]] = mul i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[VAL:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT]] to <vscale x 4 x i16>
@@ -65,16 +65,16 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len)
 ; CHECK-LABEL: @trunc_minimal_bitwidths_shufflevector(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[LEN:%.*]], [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[LEN]], [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[LEN]], [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP9:%.*]] = mul i64 [[TMP8]], 4
+; CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 4
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <vscale x 4 x i32> poison, i32 [[ARG1:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <vscale x 4 x i32> [[BROADCAST_SPLATINSERT]], <vscale x 4 x i32> poison, <vscale x 4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP4:%.*]] = trunc <vscale x 4 x i32> [[BROADCAST_SPLAT]] to <vscale x 4 x i8>
diff --git a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
index 18cb2257e7ec..17979e5f08a7 100644
--- a/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
+++ b/llvm/test/Transforms/LoopVectorize/vectorize-force-tail-with-evl.ll
@@ -37,16 +37,16 @@ define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; NO-VP-LABEL: @foo(
 ; NO-VP-NEXT:  entry:
 ; NO-VP-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP8:%.*]] = mul i64 [[TMP0]], 4
+; NO-VP-NEXT:    [[TMP8:%.*]] = mul nuw i64 [[TMP0]], 4
 ; NO-VP-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], [[TMP8]]
 ; NO-VP-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; NO-VP:       vector.ph:
 ; NO-VP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP14:%.*]] = mul i64 [[TMP1]], 4
+; NO-VP-NEXT:    [[TMP14:%.*]] = mul nuw i64 [[TMP1]], 4
 ; NO-VP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP14]]
 ; NO-VP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]]
 ; NO-VP-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; NO-VP-NEXT:    [[TMP15:%.*]] = mul i64 [[TMP2]], 4
+; NO-VP-NEXT:    [[TMP15:%.*]] = mul nuw i64 [[TMP2]], 4
 ; NO-VP-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; NO-VP:       vector.body:
 ; NO-VP-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/MemCpyOpt/vscale-crashes.ll b/llvm/test/Transforms/MemCpyOpt/vscale-crashes.ll
index 0f26a08db456..80f2977fcb3a 100644
--- a/llvm/test/Transforms/MemCpyOpt/vscale-crashes.ll
+++ b/llvm/test/Transforms/MemCpyOpt/vscale-crashes.ll
@@ -102,7 +102,7 @@ define void @memmove_vector(ptr %a, ptr %b) {
 define void @memmove_agg1(ptr %a, ptr %b) {
 ; CHECK-LABEL: @memmove_agg1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 8
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 1 [[B:%.*]], ptr align 1 [[A:%.*]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
@@ -114,7 +114,7 @@ define void @memmove_agg1(ptr %a, ptr %b) {
 define void @memmove_agg2(ptr %a, ptr %b) {
 ; CHECK-LABEL: @memmove_agg2(
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 16
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 16
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 1 [[B:%.*]], ptr align 1 [[A:%.*]], i64 [[TMP2]], i1 false)
 ; CHECK-NEXT:    ret void
 ;
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
index 6993fb281bfe..f40afbda1246 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/sve-interleave-vectorization.ll
@@ -16,16 +16,16 @@ define void @interleave_deinterleave(ptr noalias %dst, ptr %a, ptr %b) {
 ; CHECK-LABEL: @interleave_deinterleave(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 1024, [[TMP1]]
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[FOR_BODY_PREHEADER:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul i64 [[TMP2]], 4
+; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 4
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP5:%.*]] = mul i64 [[TMP4]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
diff --git a/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-exp.ll b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-exp.ll
index 284f2ad8072f..9acc6d660129 100644
--- a/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-exp.ll
+++ b/llvm/test/Transforms/PreISelIntrinsicLowering/AArch64/expand-exp.ll
@@ -7,7 +7,7 @@ define <vscale x 4 x float> @scalable_vec_exp(<vscale x 4 x float> %input) {
 ; CHECK-LABEL: define <vscale x 4 x float> @scalable_vec_exp(
 ; CHECK-SAME: <vscale x 4 x float> [[INPUT:%.*]]) {
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
 ; CHECK-NEXT:    br label %[[BB3:.*]]
 ; CHECK:       [[BB3]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = phi i64 [ 0, [[TMP0:%.*]] ], [ [[TMP9:%.*]], %[[BB3]] ]
-- 
cgit v1.2.3


From c4c2d777f4aea07c59ff85ade75816df24b05389 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 19 Jun 2025 13:26:19 +0100
Subject: [VPlan] Fix handling of ReductionStartVector for rdxs when unrolling.

Update handling of ReductionStartVector in VPlanUnroll for partial
reductions. The new code makes sure all parts are properly set to the
cloned ReductionStartVector.

Fixes a mis-compile reported for
https://github.com/llvm/llvm-project/pull/142290.
---
 llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp                 | 11 ++++++++---
 .../LoopVectorize/AArch64/partial-reduce-interleave.ll        |  8 ++++----
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
index dfb5bfabd22b..0bc683e557e7 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
@@ -231,14 +231,19 @@ void UnrollState::unrollHeaderPHIByUF(VPHeaderPHIRecipe *R,
       if (auto *VPI = dyn_cast<VPInstruction>(RdxPhi->getStartValue())) {
         assert(VPI->getOpcode() == VPInstruction::ReductionStartVector &&
                "unexpected start VPInstruction");
+        if (Part != 1)
+          continue;
+        VPValue *StartV;
         if (match(VPI->getOperand(2), m_SpecificInt(1))) {
-          Copy->setOperand(0, VPI->getOperand(1));
-        } else if (Part == 1) {
+          StartV = VPI->getOperand(1);
+        } else {
           auto *C = VPI->clone();
           C->setOperand(0, C->getOperand(1));
           C->insertAfter(VPI);
-          addUniformForAllParts(C);
+          StartV = C;
         }
+        for (unsigned Part = 1; Part != UF; ++Part)
+          VPV2Parts[VPI][Part - 1] = StartV;
       }
       Copy->addOperand(getConstantVPV(Part));
     } else {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll
index b5bc7bf80372..3515365c7027 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/partial-reduce-interleave.ll
@@ -18,7 +18,7 @@ define i32 @partial_reduce_with_non_constant_start_value(ptr %src, i32 %rdx.star
 ; IC2:       [[VECTOR_BODY]]:
 ; IC2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; IC2-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; IC2-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
+; IC2-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE3:%.*]], %[[VECTOR_BODY]] ]
 ; IC2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
 ; IC2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; IC2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
@@ -70,9 +70,9 @@ define i32 @partial_reduce_with_non_constant_start_value(ptr %src, i32 %rdx.star
 ; IC4:       [[VECTOR_BODY]]:
 ; IC4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
 ; IC4-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE:%.*]], %[[VECTOR_BODY]] ]
-; IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
-; IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
-; IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ [[TMP0]], %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI1:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE7:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI2:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE8:%.*]], %[[VECTOR_BODY]] ]
+; IC4-NEXT:    [[VEC_PHI3:%.*]] = phi <4 x i32> [ zeroinitializer, %[[VECTOR_PH]] ], [ [[PARTIAL_REDUCE9:%.*]], %[[VECTOR_BODY]] ]
 ; IC4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[INDEX]]
 ; IC4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0
 ; IC4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 16
-- 
cgit v1.2.3


From 5148e085386fb1808fba055e170d88e3344220ca Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Thu, 19 Jun 2025 13:30:20 +0100
Subject: Fix build issue caused by commit #0fe78c4 (#144888)

Noticed internally in blaze build.
---
 clang/include/clang/Serialization/ASTRecordWriter.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/clang/include/clang/Serialization/ASTRecordWriter.h b/clang/include/clang/Serialization/ASTRecordWriter.h
index ad1ec2673812..964c9e6ea8a2 100644
--- a/clang/include/clang/Serialization/ASTRecordWriter.h
+++ b/clang/include/clang/Serialization/ASTRecordWriter.h
@@ -17,6 +17,7 @@
 #include "clang/AST/AbstractBasicWriter.h"
 #include "clang/AST/OpenACCClause.h"
 #include "clang/AST/OpenMPClause.h"
+#include "clang/Serialization/ASTReader.h"
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Serialization/SourceLocationEncoding.h"
 
-- 
cgit v1.2.3


From dae5104eed451fdd0354ff9639feba10f9dc5440 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Thu, 19 Jun 2025 13:42:23 +0100
Subject: [lldb][DWARFASTParserClang] Make GetCXXObjectParameter public and
 call it from unit-tests (#144879)

My goal is to remove the `object_pointer` member on
`ParsedDWARFTypeAttributes` since it's duplicating information that we
retrieve with `GetCXXObjectParameter` anyway. To continue having
coverage for the `DW_AT_object_pointer` code-paths, instead of checking
the
`attrs.object_pointer` I'm now calling `GetCXXObjectParameter` directly.
We could find some very roundabout way to go via the Clang AST to check
that the object parameter was parsed correctly, but that quickly became
quite painful.

Depends on https://github.com/llvm/llvm-project/pull/144876
---
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp       |  5 +--
 .../Plugins/SymbolFile/DWARF/DWARFASTParserClang.h |  4 +++
 .../SymbolFile/DWARF/DWARFASTParserClangTests.cpp  | 38 +++++++++++++++-------
 3 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 7fc1d70898d1..4f79c8aa3f81 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -163,8 +163,9 @@ static bool TagIsRecordType(dw_tag_t tag) {
 /// a default DWARFDIE. If \c containing_decl_ctx is not a valid
 /// C++ declaration context for class methods, assume no object
 /// parameter exists for the given \c subprogram.
-static DWARFDIE GetCXXObjectParameter(const DWARFDIE &subprogram,
-                                      const DWARFDIE &decl_ctx_die) {
+DWARFDIE
+DWARFASTParserClang::GetCXXObjectParameter(const DWARFDIE &subprogram,
+                                           const DWARFDIE &decl_ctx_die) {
   assert(subprogram);
   assert(subprogram.Tag() == DW_TAG_subprogram ||
          subprogram.Tag() == DW_TAG_inlined_subroutine ||
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 3994726aa6b3..111604ce4068 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -112,6 +112,10 @@ public:
   void MapDeclDIEToDefDIE(const lldb_private::plugin::dwarf::DWARFDIE &decl_die,
                           const lldb_private::plugin::dwarf::DWARFDIE &def_die);
 
+  lldb_private::plugin::dwarf::DWARFDIE GetCXXObjectParameter(
+      const lldb_private::plugin::dwarf::DWARFDIE &subprogram,
+      const lldb_private::plugin::dwarf::DWARFDIE &decl_ctx_die);
+
 protected:
   /// Protected typedefs and members.
   /// @{
diff --git a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
index 6c77736113da..2d4b79fed4a5 100644
--- a/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
+++ b/lldb/unittests/SymbolFile/DWARF/DWARFASTParserClangTests.cpp
@@ -889,18 +889,32 @@ DWARF:
   ASSERT_TRUE(context_die.IsValid());
   ASSERT_EQ(context_die.Tag(), DW_TAG_structure_type);
 
-  auto subprogram_definition = context_die.GetSibling();
-  ASSERT_TRUE(subprogram_definition.IsValid());
-  ASSERT_EQ(subprogram_definition.Tag(), DW_TAG_subprogram);
-  ASSERT_FALSE(subprogram_definition.GetAttributeValueAsOptionalUnsigned(
-      DW_AT_external));
-
-  auto param_die = subprogram_definition.GetFirstChild();
-  ASSERT_TRUE(param_die.IsValid());
-
-  ParsedDWARFTypeAttributes attrs(subprogram_definition);
-  EXPECT_TRUE(attrs.object_pointer.IsValid());
-  EXPECT_EQ(attrs.object_pointer, param_die);
+  {
+    auto decl_die = context_die.GetFirstChild();
+    ASSERT_TRUE(decl_die.IsValid());
+    ASSERT_EQ(decl_die.Tag(), DW_TAG_subprogram);
+    ASSERT_TRUE(decl_die.GetAttributeValueAsOptionalUnsigned(DW_AT_external));
+
+    auto param_die = decl_die.GetFirstChild();
+    ASSERT_TRUE(param_die.IsValid());
+
+    EXPECT_EQ(param_die,
+              ast_parser.GetCXXObjectParameter(decl_die, context_die));
+  }
+
+  {
+    auto subprogram_definition = context_die.GetSibling();
+    ASSERT_TRUE(subprogram_definition.IsValid());
+    ASSERT_EQ(subprogram_definition.Tag(), DW_TAG_subprogram);
+    ASSERT_FALSE(subprogram_definition.GetAttributeValueAsOptionalUnsigned(
+        DW_AT_external));
+
+    auto param_die = subprogram_definition.GetFirstChild();
+    ASSERT_TRUE(param_die.IsValid());
+
+    EXPECT_EQ(param_die, ast_parser.GetCXXObjectParameter(subprogram_definition,
+                                                          context_die));
+  }
 }
 
 TEST_F(DWARFASTParserClangTests, TestParseSubroutine_ExplicitObjectParameter) {
-- 
cgit v1.2.3


From c079040eea5ce75a97285003948d141ebaac69e6 Mon Sep 17 00:00:00 2001
From: Charles Zablit <c_zablit@apple.com>
Date: Thu, 19 Jun 2025 13:44:41 +0100
Subject: [lldb] add has methods to all DemangledNameInfo attributes (#144549)

Add `hasX` methods to all the attributes of `DemangledNameInfo`.
---
 lldb/include/lldb/Core/DemangledNameInfo.h | 19 ++++++++
 lldb/unittests/Core/MangledTest.cpp        | 76 ++++++++++++++++++++++++++++++
 2 files changed, 95 insertions(+)

diff --git a/lldb/include/lldb/Core/DemangledNameInfo.h b/lldb/include/lldb/Core/DemangledNameInfo.h
index 4b5ba5e42b3b..a2f3fde90c61 100644
--- a/lldb/include/lldb/Core/DemangledNameInfo.h
+++ b/lldb/include/lldb/Core/DemangledNameInfo.h
@@ -73,6 +73,25 @@ struct DemangledNameInfo {
   bool hasBasename() const {
     return BasenameRange.second > BasenameRange.first;
   }
+
+  /// Returns \c true if this object holds a valid scope range.
+  bool hasScope() const { return ScopeRange.second > ScopeRange.first; }
+
+  /// Returns \c true if this object holds a valid arguments range.
+  bool hasArguments() const {
+    return ArgumentsRange.second > ArgumentsRange.first;
+  }
+
+  /// Returns \c true if this object holds a valid qualifiers range.
+  bool hasQualifiers() const {
+    return QualifiersRange.second > QualifiersRange.first;
+  }
+
+  /// Returns \c true if this object holds a valid prefix range.
+  bool hasPrefix() const { return PrefixRange.second > PrefixRange.first; }
+
+  /// Returns \c true if this object holds a valid suffix range.
+  bool hasSuffix() const { return SuffixRange.second > SuffixRange.first; }
 };
 
 /// An OutputBuffer which keeps a record of where certain parts of a
diff --git a/lldb/unittests/Core/MangledTest.cpp b/lldb/unittests/Core/MangledTest.cpp
index 46adb6272209..5994d6072481 100644
--- a/lldb/unittests/Core/MangledTest.cpp
+++ b/lldb/unittests/Core/MangledTest.cpp
@@ -635,6 +635,82 @@ TEST_P(DemanglingPartsTestFixture, DemanglingParts) {
 INSTANTIATE_TEST_SUITE_P(DemanglingPartsTests, DemanglingPartsTestFixture,
                          ::testing::ValuesIn(g_demangling_parts_test_cases));
 
+struct DemangledNameInfoTestCase {
+  DemangledNameInfo expected_info;
+  bool valid_basename;
+  bool valid_scope;
+  bool valid_arguments;
+  bool valid_qualifiers;
+  bool valid_prefix;
+  bool valid_suffix;
+};
+
+DemangledNameInfoTestCase g_demangled_name_info_test_cases[] = {
+    // clang-format off
+   {
+    { /*.BasenameRange=*/{0, 10}, /*.ScopeRange=*/{0, 0}, /*.ArgumentsRange=*/{0, 0},
+      /*.QualifiersRange=*/{0, 0}, /*.PrefixRange=*/{0, 0}, /*.SuffixRange=*/{0, 0}
+    },
+      /*valid_basename=*/true, /*valid_scope=*/false, /*valid_arguments=*/false,
+      /*valid_qualifiers=*/false, /*valid_prefix=*/false, /*valid_suffix=*/false,
+   },
+   {
+    { /*.BasenameRange=*/{0, 0}, /*.ScopeRange=*/{0, 10}, /*.ArgumentsRange=*/{0, 0},
+      /*.QualifiersRange=*/{0, 0}, /*.PrefixRange=*/{0, 0}, /*.SuffixRange=*/{0, 0}
+    },
+      /*valid_basename=*/false, /*valid_scope=*/true, /*valid_arguments=*/false,
+      /*valid_qualifiers=*/false, /*valid_prefix=*/false, /*valid_suffix=*/false,
+   },
+   {
+    { /*.BasenameRange=*/{0, 0}, /*.ScopeRange=*/{0, 0}, /*.ArgumentsRange=*/{0, 10},
+      /*.QualifiersRange=*/{0, 0}, /*.PrefixRange=*/{0, 0}, /*.SuffixRange=*/{0, 0}
+    },
+      /*valid_basename=*/false, /*valid_scope=*/false, /*valid_arguments=*/true,
+      /*valid_qualifiers=*/false, /*valid_prefix=*/false, /*valid_suffix=*/false,
+   },
+   {
+    { /*.BasenameRange=*/{0, 0}, /*.ScopeRange=*/{0, 0}, /*.ArgumentsRange=*/{0, 0},
+      /*.QualifiersRange=*/{0, 10}, /*.PrefixRange=*/{0, 0}, /*.SuffixRange=*/{0, 0}
+    },
+      /*valid_basename=*/false, /*valid_scope=*/false, /*valid_arguments=*/false,
+      /*valid_qualifiers=*/true, /*valid_prefix=*/false, /*valid_suffix=*/false,
+   },
+   {
+    { /*.BasenameRange=*/{0, 0}, /*.ScopeRange=*/{0, 0}, /*.ArgumentsRange=*/{0, 0},
+      /*.QualifiersRange=*/{0, 0}, /*.PrefixRange=*/{0, 10}, /*.SuffixRange=*/{0, 0}
+    },
+      /*valid_basename=*/false, /*valid_scope=*/false, /*valid_arguments=*/false,
+      /*valid_qualifiers=*/false, /*valid_prefix=*/true, /*valid_suffix=*/false,
+   },
+   {
+    { /*.BasenameRange=*/{0, 0}, /*.ScopeRange=*/{0, 0}, /*.ArgumentsRange=*/{0, 0},
+      /*.QualifiersRange=*/{0, 0}, /*.PrefixRange=*/{0, 0}, /*.SuffixRange=*/{0, 10}
+    },
+      /*valid_basename=*/false, /*valid_scope=*/false, /*valid_arguments=*/false,
+      /*valid_qualifiers=*/false, /*valid_prefix=*/false, /*valid_suffix=*/true,
+   },
+    // clang-format on
+};
+
+struct DemangledNameInfoTestFixture
+    : public ::testing::TestWithParam<DemangledNameInfoTestCase> {};
+
+TEST_P(DemangledNameInfoTestFixture, DemangledNameInfoRanges) {
+  const auto &[info, valid_basename, valid_scope, valid_arguments,
+               valid_qualifiers, valid_prefix, valid_suffix] = GetParam();
+
+  ASSERT_EQ(info.hasBasename(), valid_basename);
+  ASSERT_EQ(info.hasScope(), valid_scope);
+  ASSERT_EQ(info.hasArguments(), valid_arguments);
+  ASSERT_EQ(info.hasQualifiers(), valid_qualifiers);
+  ASSERT_EQ(info.hasPrefix(), valid_prefix);
+  ASSERT_EQ(info.hasSuffix(), valid_suffix);
+}
+
+INSTANTIATE_TEST_SUITE_P(DemangledNameInfoRangesTests,
+                         DemangledNameInfoTestFixture,
+                         ::testing::ValuesIn(g_demangled_name_info_test_cases));
+
 struct DemanglingInfoCorrectnessTestCase {
   const char *mangled;
   const char *demangled;
-- 
cgit v1.2.3


From 09e794c4bb138e14b3156d7dbdac0164d9c0327b Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 19 Jun 2025 07:47:03 -0500
Subject: [HIP] Emit the CUID value in the module with the new driver (#144570)

Summary:
This is a weird point of divergence that was not updated when the new
driver
switched to using the CUID method, which is also apparently critical
for SPIR-V compilation not failing? Somehow if we don't emit this global
than the `llvm.compiler.used` global uses AS(0) which makes SPIR-V
unhappy, but with this global it's AS(4) which makes it happy. Either
way, this should be fixed.
---
 clang/lib/CodeGen/CodeGenModule.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index c27168e4c4bf..16688810d068 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -969,7 +969,7 @@ void CodeGenModule::Release() {
         llvm::ConstantArray::get(ATy, UsedArray), "__clang_gpu_used_external");
     addCompilerUsedGlobal(GV);
   }
-  if (LangOpts.HIP && !getLangOpts().OffloadingNewDriver) {
+  if (LangOpts.HIP) {
     // Emit a unique ID so that host and device binaries from the same
     // compilation unit can be associated.
     auto *GV = new llvm::GlobalVariable(
-- 
cgit v1.2.3


From e873fd157eda617ffd42edad3c4a6ab495e6e375 Mon Sep 17 00:00:00 2001
From: Anatoly Trosinenko <atrosinenko@accesssoftek.com>
Date: Thu, 19 Jun 2025 15:52:54 +0300
Subject: [BOLT] Gadget scanner: do not crash on debug-printing CFI
 instructions (#136151)

Some instruction-printing code used under LLVM_DEBUG does not handle CFI
instructions well. While CFI instructions seem to be harmless for the
correctness of the analysis results, they do not convey any useful
information to the analysis either, so skip them early.
---
 bolt/lib/Passes/PAuthGadgetScanner.cpp             | 16 +++++++++++
 .../AArch64/gs-pauth-debug-output.s                | 32 ++++++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/bolt/lib/Passes/PAuthGadgetScanner.cpp b/bolt/lib/Passes/PAuthGadgetScanner.cpp
index 7682d7fe2c54..95e831fe9c8c 100644
--- a/bolt/lib/Passes/PAuthGadgetScanner.cpp
+++ b/bolt/lib/Passes/PAuthGadgetScanner.cpp
@@ -430,6 +430,9 @@ protected:
   }
 
   SrcState computeNext(const MCInst &Point, const SrcState &Cur) {
+    if (BC.MIB->isCFI(Point))
+      return Cur;
+
     SrcStatePrinter P(BC);
     LLVM_DEBUG({
       dbgs() << "  SrcSafetyAnalysis::ComputeNext(";
@@ -704,6 +707,8 @@ public:
     SrcState S = createEntryState();
     for (auto &I : BF.instrs()) {
       MCInst &Inst = I.second;
+      if (BC.MIB->isCFI(Inst))
+        continue;
 
       // If there is a label before this instruction, it is possible that it
       // can be jumped-to, thus conservatively resetting S. As an exception,
@@ -1010,6 +1015,9 @@ protected:
   }
 
   DstState computeNext(const MCInst &Point, const DstState &Cur) {
+    if (BC.MIB->isCFI(Point))
+      return Cur;
+
     DstStatePrinter P(BC);
     LLVM_DEBUG({
       dbgs() << "  DstSafetyAnalysis::ComputeNext(";
@@ -1177,6 +1185,8 @@ public:
     DstState S = createUnsafeState();
     for (auto &I : llvm::reverse(BF.instrs())) {
       MCInst &Inst = I.second;
+      if (BC.MIB->isCFI(Inst))
+        continue;
 
       // If Inst can change the control flow, we cannot be sure that the next
       // instruction (to be executed in analyzed program) is the one processed
@@ -1366,6 +1376,9 @@ void FunctionAnalysisContext::findUnsafeUses(
   });
 
   iterateOverInstrs(BF, [&](MCInstReference Inst) {
+    if (BC.MIB->isCFI(Inst))
+      return;
+
     const SrcState &S = Analysis->getStateBefore(Inst);
 
     // If non-empty state was never propagated from the entry basic block
@@ -1429,6 +1442,9 @@ void FunctionAnalysisContext::findUnsafeDefs(
   });
 
   iterateOverInstrs(BF, [&](MCInstReference Inst) {
+    if (BC.MIB->isCFI(Inst))
+      return;
+
     const DstState &S = Analysis->getStateAfter(Inst);
 
     if (auto Report = shouldReportAuthOracle(BC, Inst, S))
diff --git a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
index 686557eb1e52..fbb96a63d41e 100644
--- a/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
+++ b/bolt/test/binary-analysis/AArch64/gs-pauth-debug-output.s
@@ -329,6 +329,38 @@ auth_oracle:
 // PAUTH-EMPTY:
 // PAUTH-NEXT:   Attaching leakage info to:     00000000:      autia   x0, x1 # DataflowDstSafetyAnalysis: dst-state<CannotEscapeUnchecked: BitVector, Insts: [0](0x{{[0-9a-f]+}} )>
 
+// Gadget scanner should not crash on CFI instructions, including when debug-printing them.
+// Note that the particular debug output is not checked, but BOLT should be
+// compiled with assertions enabled to support -debug-only argument.
+
+        .globl  cfi_inst_df
+        .type   cfi_inst_df,@function
+cfi_inst_df:
+        .cfi_startproc
+        sub     sp, sp, #16
+        .cfi_def_cfa_offset 16
+        add     sp, sp, #16
+        .cfi_def_cfa_offset 0
+        ret
+        .size   cfi_inst_df, .-cfi_inst_df
+        .cfi_endproc
+
+        .globl  cfi_inst_nocfg
+        .type   cfi_inst_nocfg,@function
+cfi_inst_nocfg:
+        .cfi_startproc
+        sub     sp, sp, #16
+        .cfi_def_cfa_offset 16
+
+        adr     x0, 1f
+        br      x0
+1:
+        add     sp, sp, #16
+        .cfi_def_cfa_offset 0
+        ret
+        .size   cfi_inst_nocfg, .-cfi_inst_nocfg
+        .cfi_endproc
+
 // CHECK-LABEL:Analyzing function main, AllocatorId = 1
         .globl  main
         .type   main,@function
-- 
cgit v1.2.3


From 493a359237e824216d5c572656481c42165a2cb7 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Thu, 19 Jun 2025 12:58:08 +0000
Subject: [lldb][AArch64] Fix live process test for Linux's mte_ctrl register

I forgot to update this when I changed the presentation of the
"TCF" field.
---
 .../register/register/aarch64_mte_ctrl_register/TestMTECtrlRegister.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lldb/test/API/commands/register/register/aarch64_mte_ctrl_register/TestMTECtrlRegister.py b/lldb/test/API/commands/register/register/aarch64_mte_ctrl_register/TestMTECtrlRegister.py
index 3eaca00b0dcc..2570f267bf46 100644
--- a/lldb/test/API/commands/register/register/aarch64_mte_ctrl_register/TestMTECtrlRegister.py
+++ b/lldb/test/API/commands/register/register/aarch64_mte_ctrl_register/TestMTECtrlRegister.py
@@ -42,8 +42,9 @@ class MTECtrlRegisterTestCase(TestBase):
             expected = [value]
 
             if self.hasXMLSupport():
+                tfc_modes = ["NONE", "SYNC", "ASYNC", "ASYMM"]
                 expected.append(
-                    "(TAGS = 0, TCF_ASYNC = {}, TCF_SYNC = {}, TAGGED_ADDR_ENABLE = 1)".format(
+                    f"(TAGS = 0, TCF = TCF_{tfc_modes[async_err << 1 | sync_err]}, TAGGED_ADDR_ENABLE = 1)".format(
                         async_err, sync_err
                     )
                 )
-- 
cgit v1.2.3


From 83381ba832a5cf34b09e27a6154c7179fed2fc80 Mon Sep 17 00:00:00 2001
From: Ilia Kuklin <ikuklin@accesssoftek.com>
Date: Thu, 19 Jun 2025 18:10:56 +0500
Subject: [LLDB] Add negative number parsing to DIL (#144557)

---
 lldb/source/ValueObject/DILParser.cpp                         | 11 +++++++++--
 .../basics/ArraySubscript/TestFrameVarDILArraySubscript.py    |  6 +-----
 .../API/commands/frame/var-dil/basics/ArraySubscript/main.cpp |  1 +
 3 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/lldb/source/ValueObject/DILParser.cpp b/lldb/source/ValueObject/DILParser.cpp
index 32af0820acb9..5abbfeabcea3 100644
--- a/lldb/source/ValueObject/DILParser.cpp
+++ b/lldb/source/ValueObject/DILParser.cpp
@@ -348,8 +348,15 @@ void DILParser::BailOut(const std::string &error, uint32_t loc,
 //    ? Integer constant ?
 //
 std::optional<int64_t> DILParser::ParseIntegerConstant() {
-  auto spelling = CurToken().GetSpelling();
-  llvm::StringRef spelling_ref = spelling;
+  std::string number_spelling;
+  if (CurToken().GetKind() == Token::minus) {
+    // StringRef::getAsInteger<>() can parse negative numbers.
+    // FIXME: Remove this once unary minus operator is added.
+    number_spelling = "-";
+    m_dil_lexer.Advance();
+  }
+  number_spelling.append(CurToken().GetSpelling());
+  llvm::StringRef spelling_ref = number_spelling;
   int64_t raw_value;
   if (!spelling_ref.getAsInteger<int64_t>(0, raw_value)) {
     m_dil_lexer.Advance();
diff --git a/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py b/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
index c90e0eaa6363..c0ef29fab859 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
+++ b/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/TestFrameVarDILArraySubscript.py
@@ -60,11 +60,7 @@ class TestFrameVarDILArraySubscript(TestBase):
         self.expect_var_path("*(&int_arr[1])", value="2")
 
         # Test for negative index.
-        self.expect(
-            "frame var 'int_arr[-1]'",
-            error=True,
-            substrs=["failed to parse integer constant"],
-        )
+        self.expect_var_path("int_ptr_1[-1]", True, value="1")
 
         # Test for floating point index
         self.expect(
diff --git a/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/main.cpp b/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/main.cpp
index 485666ae46c2..a9a3612dfae5 100644
--- a/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/main.cpp
+++ b/lldb/test/API/commands/frame/var-dil/basics/ArraySubscript/main.cpp
@@ -3,6 +3,7 @@
 int main(int argc, char **argv) {
   int int_arr[] = {1, 2, 3};
   int *int_ptr = int_arr;
+  int *int_ptr_1 = &int_arr[1];
   int(&int_arr_ref)[3] = int_arr;
   void *p_void = (void *)int_arr;
 
-- 
cgit v1.2.3


From 5645d6710904107d66a45f1c3ee0ee25924ff08a Mon Sep 17 00:00:00 2001
From: Aly ElAshram <71949028+AlyElashram@users.noreply.github.com>
Date: Thu, 19 Jun 2025 15:15:59 +0200
Subject: Implement `sigsetjmp` and `siglongjmp` for darwin/aarch64 (#139555)

---
 libc/config/darwin/aarch64/config.json        |  8 ++++++++
 libc/config/darwin/aarch64/entrypoints.txt    | 11 +++++++++++
 libc/config/darwin/aarch64/headers.txt        |  1 +
 libc/src/setjmp/CMakeLists.txt                | 10 ++++++----
 libc/src/setjmp/darwin/CMakeLists.txt         | 12 ++++++++++++
 libc/src/setjmp/darwin/sigsetjmp_epilogue.cpp | 21 +++++++++++++++++++++
 libc/test/src/CMakeLists.txt                  |  2 +-
 7 files changed, 60 insertions(+), 5 deletions(-)
 create mode 100644 libc/config/darwin/aarch64/config.json
 create mode 100644 libc/src/setjmp/darwin/CMakeLists.txt
 create mode 100644 libc/src/setjmp/darwin/sigsetjmp_epilogue.cpp

diff --git a/libc/config/darwin/aarch64/config.json b/libc/config/darwin/aarch64/config.json
new file mode 100644
index 000000000000..c82f13e5cbf7
--- /dev/null
+++ b/libc/config/darwin/aarch64/config.json
@@ -0,0 +1,8 @@
+{
+  "setjmp": {
+    "LIBC_CONF_SETJMP_AARCH64_RESTORE_PLATFORM_REGISTER": {
+      "value": false,
+      "doc": "Avoid setjmp saving the value of x18, and longjmp restoring it. The Apple AArch64 ABI specifies that this register is reserved and should not be used"
+    }
+  }
+}
diff --git a/libc/config/darwin/aarch64/entrypoints.txt b/libc/config/darwin/aarch64/entrypoints.txt
index 308fc49d681d..437eca79a76f 100644
--- a/libc/config/darwin/aarch64/entrypoints.txt
+++ b/libc/config/darwin/aarch64/entrypoints.txt
@@ -101,6 +101,17 @@ set(TARGET_LIBC_ENTRYPOINTS
     libc.src.stdlib.free
 )
 
+if(LLVM_LIBC_FULL_BUILD)
+  list(APPEND TARGET_LIBC_ENTRYPOINTS
+    # setjmp.h entrypoints
+    libc.src.setjmp.longjmp
+    libc.src.setjmp.setjmp
+    libc.src.setjmp.siglongjmp
+    libc.src.setjmp.sigsetjmp
+  )
+endif()
+
+
 set(TARGET_LIBM_ENTRYPOINTS
     # complex.h entrypoints
     libc.src.complex.creal
diff --git a/libc/config/darwin/aarch64/headers.txt b/libc/config/darwin/aarch64/headers.txt
index 86e714597232..8f3d6029c9b6 100644
--- a/libc/config/darwin/aarch64/headers.txt
+++ b/libc/config/darwin/aarch64/headers.txt
@@ -7,6 +7,7 @@ set(TARGET_PUBLIC_HEADERS
     libc.include.inttypes
     libc.include.limits
     libc.include.math
+    libc.include.setjmp
     libc.include.stdlib
     libc.include.string
     libc.include.strings
diff --git a/libc/src/setjmp/CMakeLists.txt b/libc/src/setjmp/CMakeLists.txt
index 239254fa57dc..50c827254da6 100644
--- a/libc/src/setjmp/CMakeLists.txt
+++ b/libc/src/setjmp/CMakeLists.txt
@@ -1,3 +1,9 @@
+# Process architecture-specific subdirectory FIRST to avoid missing targets.
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
+endif()
+
+# Then process OS-specific subdirectory
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   add_object_library(
@@ -8,10 +14,6 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   )
 endif()
 
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
-endif()
-
 add_entrypoint_object(
   setjmp
   ALIAS
diff --git a/libc/src/setjmp/darwin/CMakeLists.txt b/libc/src/setjmp/darwin/CMakeLists.txt
new file mode 100644
index 000000000000..b844c8c5ee55
--- /dev/null
+++ b/libc/src/setjmp/darwin/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_object_library(
+  sigsetjmp_epilogue
+  HDRS
+    ../sigsetjmp_epilogue.h
+  SRCS
+    sigsetjmp_epilogue.cpp
+  DEPENDS
+    libc.src.__support.common
+    libc.src.__support.OSUtil.osutil
+    libc.hdr.types.jmp_buf
+    libc.hdr.types.sigset_t
+)
diff --git a/libc/src/setjmp/darwin/sigsetjmp_epilogue.cpp b/libc/src/setjmp/darwin/sigsetjmp_epilogue.cpp
new file mode 100644
index 000000000000..b2ca4d99ed82
--- /dev/null
+++ b/libc/src/setjmp/darwin/sigsetjmp_epilogue.cpp
@@ -0,0 +1,21 @@
+//===-- Implementation of sigsetjmp_epilogue ------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/setjmp/sigsetjmp_epilogue.h"
+#include "src/__support/OSUtil/syscall.h"
+#include "src/__support/common.h"
+#include "src/signal/sigprocmask.h"
+
+namespace LIBC_NAMESPACE_DECL {
+[[gnu::returns_twice]] int sigsetjmp_epilogue(jmp_buf buffer, int retval) {
+  syscall_impl<long>(sigprocmask, SIG_SETMASK,
+                     /* set= */ retval ? &buffer->sigmask : nullptr,
+                     /* old_set= */ retval ? nullptr : &buffer->sigmask);
+  return retval;
+}
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/test/src/CMakeLists.txt b/libc/test/src/CMakeLists.txt
index b7c145788c0c..6dca47b5343e 100644
--- a/libc/test/src/CMakeLists.txt
+++ b/libc/test/src/CMakeLists.txt
@@ -62,6 +62,7 @@ add_subdirectory(errno)
 add_subdirectory(fenv)
 add_subdirectory(math)
 add_subdirectory(search)
+add_subdirectory(setjmp)
 add_subdirectory(stdbit)
 add_subdirectory(stdfix)
 add_subdirectory(stdio)
@@ -92,7 +93,6 @@ add_subdirectory(assert)
 add_subdirectory(compiler)
 add_subdirectory(dirent)
 add_subdirectory(locale)
-add_subdirectory(setjmp)
 add_subdirectory(signal)
 add_subdirectory(spawn)
 
-- 
cgit v1.2.3


From bf79d4819edeb54c6cf528db63676110992908a8 Mon Sep 17 00:00:00 2001
From: zhijian lin <zhijian@ca.ibm.com>
Date: Thu, 19 Jun 2025 09:22:16 -0400
Subject: [Reland] [PowerPC] frontend get target feature from backend with cpu
 name (#144594)

1. The PR proceeds with a backend target hook to allow front-ends to
determine what target features are available in a compilation based on
the CPU name.
2. Fix a backend target feature bug that supports HTM for
Power8/9/10/11. However, HTM is only supported on Power8/9 according to
the ISA.
3. All target features that are hardcoded in PPC.cpp can be retrieved
from the backend target feature. I have double-checked that the
hardcoded logic for inferring target features from the CPU in the
frontend(PPC.cpp) is the same as in PPC.td.

The reland patch addressed the comment
https://github.com/llvm/llvm-project/pull/137670#discussion_r2143541120
---
 clang/lib/Basic/Targets/PPC.cpp                    | 148 +----------------
 .../CodeGenCXX/cxx11-thread-local-reference.cpp    |   2 +-
 clang/test/Driver/aix-shared-lib-tls-model-opt.c   |   7 +-
 .../test/Driver/aix-small-local-exec-dynamic-tls.c |  39 +++--
 clang/test/Driver/ppc-crbits.cpp                   |   4 -
 clang/test/Driver/ppc-isa-features.cpp             |  22 +--
 llvm/include/llvm/TargetParser/CMakeLists.txt      |   3 +
 llvm/include/llvm/TargetParser/PPCTargetParser.h   |   6 +
 llvm/include/llvm/TargetParser/TargetParser.h      |  27 ++++
 llvm/lib/Target/PowerPC/PPC.td                     |   4 +-
 llvm/lib/TargetParser/PPCTargetParser.cpp          |  25 +++
 llvm/lib/TargetParser/TargetParser.cpp             |  47 ++++++
 llvm/utils/TableGen/Basic/CMakeLists.txt           |   1 +
 .../utils/TableGen/Basic/TargetFeaturesEmitter.cpp | 178 +++++++++++++++++++++
 llvm/utils/TableGen/Basic/TargetFeaturesEmitter.h  |  49 ++++++
 llvm/utils/TableGen/SubtargetEmitter.cpp           |  84 +---------
 16 files changed, 383 insertions(+), 263 deletions(-)
 create mode 100644 llvm/utils/TableGen/Basic/TargetFeaturesEmitter.cpp
 create mode 100644 llvm/utils/TableGen/Basic/TargetFeaturesEmitter.h

diff --git a/clang/lib/Basic/Targets/PPC.cpp b/clang/lib/Basic/Targets/PPC.cpp
index e6ef0ecc526b..77145e2891a8 100644
--- a/clang/lib/Basic/Targets/PPC.cpp
+++ b/clang/lib/Basic/Targets/PPC.cpp
@@ -15,6 +15,7 @@
 #include "clang/Basic/MacroBuilder.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "llvm/TargetParser/PPCTargetParser.h"
+#include <optional>
 
 using namespace clang;
 using namespace clang::targets;
@@ -516,129 +517,14 @@ static bool ppcUserFeaturesCheck(DiagnosticsEngine &Diags,
 bool PPCTargetInfo::initFeatureMap(
     llvm::StringMap<bool> &Features, DiagnosticsEngine &Diags, StringRef CPU,
     const std::vector<std::string> &FeaturesVec) const {
-  Features["altivec"] = llvm::StringSwitch<bool>(CPU)
-                            .Case("7400", true)
-                            .Case("g4", true)
-                            .Case("7450", true)
-                            .Case("g4+", true)
-                            .Case("970", true)
-                            .Case("g5", true)
-                            .Case("pwr6", true)
-                            .Case("pwr7", true)
-                            .Case("pwr8", true)
-                            .Case("pwr9", true)
-                            .Case("ppc64", true)
-                            .Case("ppc64le", true)
-                            .Default(false);
-
-  Features["power9-vector"] = (CPU == "pwr9");
-  Features["crypto"] = llvm::StringSwitch<bool>(CPU)
-                           .Case("ppc64le", true)
-                           .Case("pwr9", true)
-                           .Case("pwr8", true)
-                           .Default(false);
-  Features["power8-vector"] = llvm::StringSwitch<bool>(CPU)
-                                  .Case("ppc64le", true)
-                                  .Case("pwr9", true)
-                                  .Case("pwr8", true)
-                                  .Default(false);
-  Features["bpermd"] = llvm::StringSwitch<bool>(CPU)
-                           .Case("ppc64le", true)
-                           .Case("pwr9", true)
-                           .Case("pwr8", true)
-                           .Case("pwr7", true)
-                           .Default(false);
-  Features["extdiv"] = llvm::StringSwitch<bool>(CPU)
-                           .Case("ppc64le", true)
-                           .Case("pwr9", true)
-                           .Case("pwr8", true)
-                           .Case("pwr7", true)
-                           .Default(false);
-  Features["direct-move"] = llvm::StringSwitch<bool>(CPU)
-                                .Case("ppc64le", true)
-                                .Case("pwr9", true)
-                                .Case("pwr8", true)
-                                .Default(false);
-  Features["crbits"] = llvm::StringSwitch<bool>(CPU)
-                                .Case("ppc64le", true)
-                                .Case("pwr9", true)
-                                .Case("pwr8", true)
-                                .Default(false);
-  Features["vsx"] = llvm::StringSwitch<bool>(CPU)
-                        .Case("ppc64le", true)
-                        .Case("pwr9", true)
-                        .Case("pwr8", true)
-                        .Case("pwr7", true)
-                        .Default(false);
-  Features["htm"] = llvm::StringSwitch<bool>(CPU)
-                        .Case("ppc64le", true)
-                        .Case("pwr9", true)
-                        .Case("pwr8", true)
-                        .Default(false);
-
-  // ROP Protect is off by default.
-  Features["rop-protect"] = false;
-  // Privileged instructions are off by default.
-  Features["privileged"] = false;
 
-  if (getTriple().isOSAIX()) {
-    // The code generated by the -maix-small-local-[exec|dynamic]-tls option is
-    // turned off by default.
-    Features["aix-small-local-exec-tls"] = false;
-    Features["aix-small-local-dynamic-tls"] = false;
-
-    // Turn off TLS model opt by default.
-    Features["aix-shared-lib-tls-model-opt"] = false;
-  }
-
-  Features["spe"] = llvm::StringSwitch<bool>(CPU)
-                        .Case("8548", true)
-                        .Case("e500", true)
-                        .Default(false);
-
-  Features["isa-v206-instructions"] = llvm::StringSwitch<bool>(CPU)
-                                          .Case("ppc64le", true)
-                                          .Case("pwr9", true)
-                                          .Case("pwr8", true)
-                                          .Case("pwr7", true)
-                                          .Case("a2", true)
-                                          .Default(false);
-
-  Features["isa-v207-instructions"] = llvm::StringSwitch<bool>(CPU)
-                                          .Case("ppc64le", true)
-                                          .Case("pwr9", true)
-                                          .Case("pwr8", true)
-                                          .Default(false);
-
-  Features["isa-v30-instructions"] =
-      llvm::StringSwitch<bool>(CPU).Case("pwr9", true).Default(false);
-
-  Features["quadword-atomics"] =
-      getTriple().isArch64Bit() && llvm::StringSwitch<bool>(CPU)
-                                       .Case("pwr9", true)
-                                       .Case("pwr8", true)
-                                       .Default(false);
-
-  // Power10 includes all the same features as Power9 plus any features specific
-  // to the Power10 core.
-  if (CPU == "pwr10" || CPU == "power10") {
-    initFeatureMap(Features, Diags, "pwr9", FeaturesVec);
-    addP10SpecificFeatures(Features);
-  }
-
-  // Power11 includes all the same features as Power10 plus any features
-  // specific to the Power11 core.
-  if (CPU == "pwr11" || CPU == "power11") {
-    initFeatureMap(Features, Diags, "pwr10", FeaturesVec);
-    addP11SpecificFeatures(Features);
-  }
+  const llvm::Triple &TheTriple = getTriple();
 
-  // Future CPU should include all of the features of Power 11 as well as any
-  // additional features (yet to be determined) specific to it.
-  if (CPU == "future") {
-    initFeatureMap(Features, Diags, "pwr11", FeaturesVec);
-    addFutureSpecificFeatures(Features);
-  }
+  std::optional<llvm::StringMap<bool>> FeaturesOpt =
+      llvm::PPC::getPPCDefaultTargetFeatures(TheTriple,
+                                             llvm::PPC::normalizeCPUName(CPU));
+  if (FeaturesOpt)
+    Features = FeaturesOpt.value();
 
   if (!ppcUserFeaturesCheck(Diags, FeaturesVec))
     return false;
@@ -700,26 +586,6 @@ bool PPCTargetInfo::initFeatureMap(
   return TargetInfo::initFeatureMap(Features, Diags, CPU, FeaturesVec);
 }
 
-// Add any Power10 specific features.
-void PPCTargetInfo::addP10SpecificFeatures(
-    llvm::StringMap<bool> &Features) const {
-  Features["htm"] = false; // HTM was removed for P10.
-  Features["paired-vector-memops"] = true;
-  Features["mma"] = true;
-  Features["power10-vector"] = true;
-  Features["pcrelative-memops"] = true;
-  Features["prefix-instrs"] = true;
-  Features["isa-v31-instructions"] = true;
-}
-
-// Add any Power11 specific features.
-void PPCTargetInfo::addP11SpecificFeatures(
-    llvm::StringMap<bool> &Features) const {}
-
-// Add features specific to the "Future" CPU.
-void PPCTargetInfo::addFutureSpecificFeatures(
-    llvm::StringMap<bool> &Features) const {}
-
 bool PPCTargetInfo::hasFeature(StringRef Feature) const {
   return llvm::StringSwitch<bool>(Feature)
       .Case("powerpc", true)
diff --git a/clang/test/CodeGenCXX/cxx11-thread-local-reference.cpp b/clang/test/CodeGenCXX/cxx11-thread-local-reference.cpp
index cd5a18f39060..a0e76e8a9a0b 100644
--- a/clang/test/CodeGenCXX/cxx11-thread-local-reference.cpp
+++ b/clang/test/CodeGenCXX/cxx11-thread-local-reference.cpp
@@ -35,5 +35,5 @@ int &g() { return r; }
 // DARWIN-LABEL: define internal cxx_fast_tlscc void @__tls_init()
 // CHECK: call void @[[R_INIT]]()
 
-// LINUX_AIX: attributes [[ATTR0]] = { {{.*}}"target-features"{{.*}} }
+// LINUX_AIX: attributes [[ATTR0]] = { {{.*}} }
 // DARWIN: attributes [[ATTR1]] = { {{.*}}nounwind{{.*}}"target-features"{{.*}}  }
diff --git a/clang/test/Driver/aix-shared-lib-tls-model-opt.c b/clang/test/Driver/aix-shared-lib-tls-model-opt.c
index 7acf091f0a04..891caf4ed3fc 100644
--- a/clang/test/Driver/aix-shared-lib-tls-model-opt.c
+++ b/clang/test/Driver/aix-shared-lib-tls-model-opt.c
@@ -1,5 +1,5 @@
-// RUN: %clang -target powerpc64-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefixes=CHECK-AIX,CHECK-AIX-OFF %s
-// RUN: %clang -target powerpc-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefixes=CHECK-AIX,CHECK-AIX-OFF %s
+// RUN: %clang -target powerpc64-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-AIX %s
+// RUN: %clang -target powerpc-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-AIX %s
 // RUN: %clang -target powerpc64le-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-LINUX %s
 // RUN: %clang -target powerpc64-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-LINUX %s
 
@@ -19,9 +19,8 @@ int test(void) {
 
 // CHECK-AIX: test() #0 {
 // CHECK-AIX: attributes #0 = {
-// CHECK-AIX-OFF-SAME: -aix-shared-lib-tls-model-opt
 // CHECK-AIX-ON-SAME: +aix-shared-lib-tls-model-opt
 
-// CHECK-LINUX-NOT: {{[-+]aix-shared-lib-tls-model-opt}}
+// CHECK-LINUX-NOT: {{[+]aix-shared-lib-tls-model-opt}}
 
 // CHECK-UNSUPPORTED-TARGET: option '-maix-shared-lib-tls-model-opt' cannot be specified on this target
diff --git a/clang/test/Driver/aix-small-local-exec-dynamic-tls.c b/clang/test/Driver/aix-small-local-exec-dynamic-tls.c
index 1a0619b58e89..6fc2b8efb4ae 100644
--- a/clang/test/Driver/aix-small-local-exec-dynamic-tls.c
+++ b/clang/test/Driver/aix-small-local-exec-dynamic-tls.c
@@ -1,37 +1,37 @@
-// RUN: %clang -target powerpc64-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-AIX-DEFAULT %s
-// RUN: %clang -target powerpc-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-AIX-DEFAULT %s
-// RUN: %clang -target powerpc64le-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-LINUX %s
-// RUN: %clang -target powerpc64-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-LINUX %s
+// RUN: %clang --target=powerpc64-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-DEFAULT %s
+// RUN: %clang --target=powerpc-unknown-aix -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-DEFAULT %s
+// RUN: %clang --target=powerpc64le-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-DEFAULT %s
+// RUN: %clang --target=powerpc64-unknown-linux-gnu -S -emit-llvm %s -o - | FileCheck --check-prefix=CHECK-DEFAULT %s
 
-// RUN: %clang -target powerpc64-unknown-aix -maix-small-local-exec-tls -S -emit-llvm \
+// RUN: %clang --target=powerpc64-unknown-aix -maix-small-local-exec-tls -S -emit-llvm \
 // RUN:    %s -o - | FileCheck %s --check-prefix=CHECK-AIX_SMALL_LOCALEXEC_TLS
 
-// RUN: %clang -target powerpc64-unknown-aix -maix-small-local-dynamic-tls -S -emit-llvm \
+// RUN: %clang --target=powerpc64-unknown-aix -maix-small-local-dynamic-tls -S -emit-llvm \
 // RUN:    %s -o - | FileCheck %s --check-prefix=CHECK-AIX_SMALL_LOCALDYNAMIC_TLS
 
-// RUN: not %clang -target powerpc-unknown-aix -maix-small-local-exec-tls \
+// RUN: not %clang --target=powerpc-unknown-aix -maix-small-local-exec-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-AIX32 %s
-// RUN: not %clang -target powerpc64le-unknown-linux-gnu -maix-small-local-exec-tls \
+// RUN: not %clang --target=powerpc64le-unknown-linux-gnu -maix-small-local-exec-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-LINUX %s
-// RUN: not %clang -target powerpc64-unknown-linux-gnu -maix-small-local-exec-tls \
+// RUN: not %clang --target=powerpc64-unknown-linux-gnu -maix-small-local-exec-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-LINUX %s
-// RUN: not %clang -target powerpc64-unknown-aix -maix-small-local-exec-tls \
+// RUN: not %clang --target=powerpc64-unknown-aix -maix-small-local-exec-tls \
 // RUN:    -fsyntax-only -fno-data-sections %s 2>&1 | \
 // RUN:    FileCheck --check-prefix=CHECK-UNSUPPORTED-NO-DATASEC %s
-// RUN: not %clang -target powerpc64-unknown-linux-gnu -maix-small-local-exec-tls \
+// RUN: not %clang --target=powerpc64-unknown-linux-gnu -maix-small-local-exec-tls \
 // RUN:    -fsyntax-only -fno-data-sections %s 2>&1 | \
 // RUN:    FileCheck --check-prefix=CHECK-UNSUPPORTED-NO-DATASEC %s
 
-// RUN: not %clang -target powerpc-unknown-aix -maix-small-local-dynamic-tls \
+// RUN: not %clang --target=powerpc-unknown-aix -maix-small-local-dynamic-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-AIX32 %s
-// RUN: not %clang -target powerpc64le-unknown-linux-gnu -maix-small-local-dynamic-tls \
+// RUN: not %clang --target=powerpc64le-unknown-linux-gnu -maix-small-local-dynamic-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-LINUX %s
-// RUN: not %clang -target powerpc64-unknown-linux-gnu -maix-small-local-dynamic-tls \
+// RUN: not %clang --target=powerpc64-unknown-linux-gnu -maix-small-local-dynamic-tls \
 // RUN:    -fsyntax-only %s 2>&1 | FileCheck --check-prefix=CHECK-UNSUPPORTED-LINUX %s
-// RUN: not %clang -target powerpc64-unknown-aix -maix-small-local-dynamic-tls \
+// RUN: not %clang --target=powerpc64-unknown-aix -maix-small-local-dynamic-tls \
 // RUN:    -fsyntax-only -fno-data-sections %s 2>&1 | \
 // RUN:    FileCheck --check-prefix=CHECK-UNSUPPORTED-NO-DATASEC %s
-// RUN: not %clang -target powerpc64-unknown-linux-gnu -maix-small-local-dynamic-tls \
+// RUN: not %clang --target=powerpc64-unknown-linux-gnu -maix-small-local-dynamic-tls \
 // RUN:    -fsyntax-only -fno-data-sections %s 2>&1 | \
 // RUN:    FileCheck --check-prefix=CHECK-UNSUPPORTED-NO-DATASEC %s
 
@@ -39,10 +39,9 @@ int test(void) {
   return 0;
 }
 
-// CHECK-AIX-DEFAULT: test() #0 {
-// CHECK-AIX-DEFAULT: attributes #0 = {
-// CHECK-AIX-DEFAULT-SAME: {{-aix-small-local-exec-tls,.*-aix-small-local-dynamic-tls|-aix-small-local-dynamic-tls,.*-aix-small-local-exec-tls}}
-// CHECK-LINUX-NOT: {{[-+]aix-small-local-exec-tls,.*[-+]aix-small-local-dynamic-tls|[-+]aix-small-local-dynamic-tls,.*[-+]aix-small-local-exec-tls}}
+// CHECK-DEFAULT: test() #0 {
+// CHECK-DEFAULT: attributes #0 = {
+// CHECK-DEFAULT-NOT: {{[-+]aix-small-local-exec-tls,.*[-+]aix-small-local-dynamic-tls|[-+]aix-small-local-dynamic-tls,.*[-+]aix-small-local-exec-tls}}
 
 // CHECK-UNSUPPORTED-AIX32: option '-maix-small-local-[exec|dynamic]-tls' cannot be specified on this target
 // CHECK-UNSUPPORTED-LINUX: option '-maix-small-local-[exec|dynamic]-tls' cannot be specified on this target
diff --git a/clang/test/Driver/ppc-crbits.cpp b/clang/test/Driver/ppc-crbits.cpp
index 3ed56308cb52..62893d3d0e87 100644
--- a/clang/test/Driver/ppc-crbits.cpp
+++ b/clang/test/Driver/ppc-crbits.cpp
@@ -64,8 +64,6 @@
 // RUN: %clang -target powerpc64le-unknown-linux-gnu -mcpu=pwr8 -mno-crbits \
 // RUN:   -emit-llvm -S %s -o - | FileCheck %s --check-prefix=HAS-NOCRBITS
 
-// RUN: %clang -target powerpc64le-unknown-linux-gnu -mcpu=pwr7 -emit-llvm \
-// RUN:   -S %s -o - | FileCheck %s --check-prefix=HAS-NOCRBITS
 // RUN: %clang -target powerpc64le-unknown-linux-gnu -mcpu=pwr7 -mcrbits \
 // RUN:   -emit-llvm -S %s -o - | FileCheck %s --check-prefix=HAS-CRBITS
 // RUN: %clang -target powerpc64le-unknown-linux-gnu -mcpu=pwr7 -mno-crbits \
@@ -92,8 +90,6 @@
 // RUN: %clang -target powerpc-ibm-aix -mcpu=pwr8 -mno-crbits \
 // RUN:   -emit-llvm -S %s -o - | FileCheck %s --check-prefix=HAS-NOCRBITS
 
-// RUN: %clang -target powerpc-ibm-aix -mcpu=pwr7 -emit-llvm \
-// RUN:   -S %s -o - | FileCheck %s --check-prefix=HAS-NOCRBITS
 // RUN: %clang -target powerpc-ibm-aix -mcpu=pwr7 -mcrbits \
 // RUN:   -emit-llvm -S %s -o - | FileCheck %s --check-prefix=HAS-CRBITS
 // RUN: %clang -target powerpc-ibm-aix -mcpu=pwr7 -mno-crbits \
diff --git a/clang/test/Driver/ppc-isa-features.cpp b/clang/test/Driver/ppc-isa-features.cpp
index 92c5bc82f72b..35dbfbcdf569 100644
--- a/clang/test/Driver/ppc-isa-features.cpp
+++ b/clang/test/Driver/ppc-isa-features.cpp
@@ -5,20 +5,20 @@
 // RUN: %clang -target powerpc64-unknown-aix -mcpu=pwr9 -S -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK-PWR9
 // RUN: %clang -target powerpc-unknown-aix -mcpu=pwr10 -S -emit-llvm %s -o - | FileCheck %s -check-prefix=CHECK-PWR10
 
-// CHECK-PWR6: -isa-v206-instructions
-// CHECK-PWR6: -isa-v207-instructions
-// CHECK-PWR6: -isa-v30-instructions
+// CHECK-PWR6-NOT: isa-v206-instructions
+// CHECK-PWR6-NOT: isa-v207-instructions
+// CHECK-PWR6-NOT: isa-v30-instructions
 
-// CHECK-A2: +isa-v206-instructions
-// CHECK-A2: -isa-v207-instructions
-// CHECK-A2: -isa-v30-instructions
+// CHECK-A2:     +isa-v206-instructions
+// CHECK-A2-NOT: isa-v207-instructions
+// CHECK-A2-NOT: isa-v30-instructions
 
-// CHECK-PWR7: +isa-v206-instructions
-// CHECK-PWR7: -isa-v207-instructions
-// CHECK-PWR7: -isa-v30-instructions
+// CHECK-PWR7:     +isa-v206-instructions
+// CHECK-PWR7-NOT: isa-v207-instructions
+// CHECK-PWR7-NOT: isa-v30-instructions
 
-// CHECK-PWR8: +isa-v207-instructions
-// CHECK-PWR8: -isa-v30-instructions
+// CHECK-PWR8:     +isa-v207-instructions
+// CHECK-PWR8-NOT: isa-v30-instructions
 
 // CHECK-PWR9: +isa-v207-instructions
 // CHECK-PWR9: +isa-v30-instructions
diff --git a/llvm/include/llvm/TargetParser/CMakeLists.txt b/llvm/include/llvm/TargetParser/CMakeLists.txt
index b456da66a022..bb6d58d74a35 100644
--- a/llvm/include/llvm/TargetParser/CMakeLists.txt
+++ b/llvm/include/llvm/TargetParser/CMakeLists.txt
@@ -7,5 +7,8 @@ tablegen(LLVM AArch64TargetParserDef.inc -gen-arm-target-def -I ${PROJECT_SOURCE
 set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/RISCV/RISCV.td)
 tablegen(LLVM RISCVTargetParserDef.inc -gen-riscv-target-def -I ${PROJECT_SOURCE_DIR}/lib/Target/RISCV/)
 
+set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/PowerPC/PPC.td)
+tablegen(LLVM PPCGenTargetFeatures.inc -gen-target-features -I${PROJECT_SOURCE_DIR}/lib/Target/PowerPC)
+
 # This covers all of the tablegen calls above.
 add_public_tablegen_target(target_parser_gen)
diff --git a/llvm/include/llvm/TargetParser/PPCTargetParser.h b/llvm/include/llvm/TargetParser/PPCTargetParser.h
index 59d9f867005a..d3d44afb5f54 100644
--- a/llvm/include/llvm/TargetParser/PPCTargetParser.h
+++ b/llvm/include/llvm/TargetParser/PPCTargetParser.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_TARGETPARSER_PPCTARGETPARSER_H
 #define LLVM_TARGETPARSER_PPCTARGETPARSER_H
 
+#include "TargetParser.h"
+#include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/TargetParser/Triple.h"
@@ -37,6 +39,10 @@ LLVM_ABI StringRef getNormalizedPPCTuneCPU(const Triple &T,
 // For PPC, there are some cpu names for same CPU, like pwr10 and power10,
 // normalize them.
 LLVM_ABI StringRef normalizeCPUName(StringRef CPUName);
+
+LLVM_ABI std::optional<llvm::StringMap<bool>>
+getPPCDefaultTargetFeatures(const Triple &T, StringRef CPUName);
+
 } // namespace PPC
 } // namespace llvm
 
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index 176205e17ae0..b4a92cc6b6c4 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -14,6 +14,8 @@
 #ifndef LLVM_TARGETPARSER_TARGETPARSER_H
 #define LLVM_TARGETPARSER_TARGETPARSER_H
 
+#include "SubtargetFeature.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
@@ -190,6 +192,31 @@ insertWaveSizeFeature(StringRef GPU, const Triple &T,
                       StringMap<bool> &Features);
 
 } // namespace AMDGPU
+
+struct BasicSubtargetFeatureKV {
+  const char *Key;         ///< K-V key string
+  unsigned Value;          ///< K-V integer value
+  FeatureBitArray Implies; ///< K-V bit mask
+};
+
+/// Used to provide key value pairs for feature and CPU bit flags.
+struct BasicSubtargetSubTypeKV {
+  const char *Key;         ///< K-V key string
+  FeatureBitArray Implies; ///< K-V bit mask
+
+  /// Compare routine for std::lower_bound
+  bool operator<(StringRef S) const { return StringRef(Key) < S; }
+
+  /// Compare routine for std::is_sorted.
+  bool operator<(const BasicSubtargetSubTypeKV &Other) const {
+    return StringRef(Key) < StringRef(Other.Key);
+  }
+};
+
+std::optional<llvm::StringMap<bool>>
+getCPUDefaultTargetFeatures(StringRef CPU,
+                            ArrayRef<BasicSubtargetSubTypeKV> ProcDesc,
+                            ArrayRef<BasicSubtargetFeatureKV> ProcFeatures);
 } // namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/PowerPC/PPC.td b/llvm/lib/Target/PowerPC/PPC.td
index fd850faf7b2f..ea7c2203662b 100644
--- a/llvm/lib/Target/PowerPC/PPC.td
+++ b/llvm/lib/Target/PowerPC/PPC.td
@@ -411,7 +411,6 @@ def ProcessorFeatures {
      FeatureP8Altivec,
      FeatureP8Vector,
      FeatureP8Crypto,
-     FeatureHTM,
      FeatureDirectMove,
      FeatureICBT,
      FeaturePartwordAtomic,
@@ -422,6 +421,7 @@ def ProcessorFeatures {
     ];
 
   list<SubtargetFeature> P8SpecificFeatures = [FeatureAddiLoadFusion,
+                                               FeatureHTM,
                                                FeatureAddisLoadFusion];
   list<SubtargetFeature> P8InheritableFeatures =
     !listconcat(P7InheritableFeatures, P8AdditionalFeatures);
@@ -443,7 +443,7 @@ def ProcessorFeatures {
   // dispatch for vector operations than scalar ones. For the time being,
   // this list also includes scheduling-related features since we do not have
   // enough info to create custom scheduling strategies for future CPUs.
-  list<SubtargetFeature> P9SpecificFeatures = [FeatureVectorsUseTwoUnits];
+  list<SubtargetFeature> P9SpecificFeatures = [FeatureVectorsUseTwoUnits, FeatureHTM];
   list<SubtargetFeature> P9InheritableFeatures =
     !listconcat(P8InheritableFeatures, P9AdditionalFeatures);
   list<SubtargetFeature> P9Features =
diff --git a/llvm/lib/TargetParser/PPCTargetParser.cpp b/llvm/lib/TargetParser/PPCTargetParser.cpp
index 422d758c772e..d51044529a49 100644
--- a/llvm/lib/TargetParser/PPCTargetParser.cpp
+++ b/llvm/lib/TargetParser/PPCTargetParser.cpp
@@ -15,6 +15,10 @@
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/TargetParser/Host.h"
 
+#define GET_SUBTARGETFEATURES_ENUM
+#define GET_SUBTARGETFEATURES_KV
+#include "llvm/TargetParser/PPCGenTargetFeatures.inc"
+
 namespace llvm {
 namespace PPC {
 
@@ -117,5 +121,26 @@ StringRef getNormalizedPPCTuneCPU(const Triple &T, StringRef CPUName) {
   return getNormalizedPPCTargetCPU(T, CPUName);
 }
 
+std::optional<StringMap<bool>> getPPCDefaultTargetFeatures(const Triple &T,
+                                                           StringRef CPU) {
+  std::optional<StringMap<bool>> FeaturesOpt =
+      getCPUDefaultTargetFeatures(CPU, BasicPPCSubTypeKV, BasicPPCFeatureKV);
+
+  if (!FeaturesOpt.has_value())
+    return std::nullopt;
+
+  StringMap<bool> Features = FeaturesOpt.value();
+  // FIXME: We need to check for the processor model 8548, since the backend
+  // does not support this processor. When this processor model is implemented
+  // within the backend, the following code can be removed.
+  if (CPU == "8548")
+    Features["spe"] = true;
+
+  // The target feature `quadword-atomics` is only supported for 64-bit
+  // POWER8 and above.
+  if (Features.find("quadword-atomics") != Features.end() && !T.isArch64Bit())
+    Features["quadword-atomics"] = false;
+  return Features;
+}
 } // namespace PPC
 } // namespace llvm
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 7c54901dae47..03f7d3899c2e 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -18,6 +18,53 @@
 using namespace llvm;
 using namespace AMDGPU;
 
+/// Find KV in array using binary search.
+static const BasicSubtargetSubTypeKV *
+find(StringRef S, ArrayRef<BasicSubtargetSubTypeKV> A) {
+  // Binary search the array
+  auto F = llvm::lower_bound(A, S);
+  // If not found then return NULL
+  if (F == A.end() || StringRef(F->Key) != S)
+    return nullptr;
+  // Return the found array item
+  return F;
+}
+
+/// For each feature that is (transitively) implied by this feature, set it.
+static void setImpliedBits(FeatureBitset &Bits, const FeatureBitset &Implies,
+                           ArrayRef<BasicSubtargetFeatureKV> FeatureTable) {
+  // OR the Implies bits in outside the loop. This allows the Implies for CPUs
+  // which might imply features not in FeatureTable to use this.
+  Bits |= Implies;
+  for (const auto &FE : FeatureTable)
+    if (Implies.test(FE.Value))
+      setImpliedBits(Bits, FE.Implies.getAsBitset(), FeatureTable);
+}
+
+std::optional<llvm::StringMap<bool>> llvm::getCPUDefaultTargetFeatures(
+    StringRef CPU, ArrayRef<BasicSubtargetSubTypeKV> ProcDesc,
+    ArrayRef<BasicSubtargetFeatureKV> ProcFeatures) {
+  if (CPU.empty())
+    return std::nullopt;
+
+  const BasicSubtargetSubTypeKV *CPUEntry = ::find(CPU, ProcDesc);
+  if (!CPUEntry)
+    return std::nullopt;
+
+  // Set the features implied by this CPU feature if there is a match.
+  FeatureBitset Bits;
+  llvm::StringMap<bool> DefaultFeatures;
+  setImpliedBits(Bits, CPUEntry->Implies.getAsBitset(), ProcFeatures);
+
+  unsigned BitSize = Bits.size();
+  for (const BasicSubtargetFeatureKV &FE : ProcFeatures) {
+    assert(FE.Value < BitSize && "Target Feature is out of range");
+    if (Bits[FE.Value])
+      DefaultFeatures[FE.Key] = true;
+  }
+  return DefaultFeatures;
+}
+
 namespace {
 
 struct GPUInfo {
diff --git a/llvm/utils/TableGen/Basic/CMakeLists.txt b/llvm/utils/TableGen/Basic/CMakeLists.txt
index b058fba78eb0..1f4d3a7bc123 100644
--- a/llvm/utils/TableGen/Basic/CMakeLists.txt
+++ b/llvm/utils/TableGen/Basic/CMakeLists.txt
@@ -17,6 +17,7 @@ add_llvm_library(LLVMTableGenBasic OBJECT EXCLUDE_FROM_ALL DISABLE_LLVM_LINK_LLV
   RISCVTargetDefEmitter.cpp
   SDNodeProperties.cpp
   TableGen.cpp
+  TargetFeaturesEmitter.cpp
   VTEmitter.cpp
 )
 
diff --git a/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.cpp b/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.cpp
new file mode 100644
index 000000000000..f016cc43c0e3
--- /dev/null
+++ b/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.cpp
@@ -0,0 +1,178 @@
+//===- TargetFeaturesEmitter.cpp - Generate CPU Target feature ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This tablegen backend exports cpu target features
+//  and cpu sub-type.
+//
+//===----------------------------------------------------------------------===//
+
+#include "TargetFeaturesEmitter.h"
+#include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/TableGenBackend.h"
+#include "llvm/TargetParser/SubtargetFeature.h"
+
+using namespace llvm;
+
+using FeatureMapTy = DenseMap<const Record *, unsigned>;
+using ConstRecVec = std::vector<const Record *>;
+
+TargetFeaturesEmitter::TargetFeaturesEmitter(const RecordKeeper &R)
+    : Records(R) {
+  ArrayRef<const Record *> Targets = Records.getAllDerivedDefinitions("Target");
+  if (Targets.size() == 0)
+    PrintFatalError("No 'Target' subclasses defined!");
+  if (Targets.size() != 1)
+    PrintFatalError("Multiple subclasses of Target defined!");
+  Target = Targets[0]->getName();
+}
+
+FeatureMapTy TargetFeaturesEmitter::enumeration(raw_ostream &OS) {
+  ArrayRef<const Record *> DefList =
+      Records.getAllDerivedDefinitions("SubtargetFeature");
+
+  unsigned N = DefList.size();
+  if (N == 0)
+    return FeatureMapTy();
+
+  if (N + 1 > MAX_SUBTARGET_FEATURES)
+    PrintFatalError(
+        "Too many subtarget features! Bump MAX_SUBTARGET_FEATURES.");
+
+  OS << "namespace " << Target << " {\n";
+
+  OS << "enum {\n";
+
+  FeatureMapTy FeatureMap;
+  for (unsigned I = 0; I < N; ++I) {
+    const Record *Def = DefList[I];
+    // Print the Feature Name.
+    OS << "  " << Def->getName() << " = " << I << ",\n";
+
+    FeatureMap[Def] = I;
+  }
+
+  OS << "  " << "NumSubtargetFeatures = " << N << "\n";
+
+  // Close enumeration and namespace
+  OS << "};\n";
+  OS << "} // end namespace " << Target << "\n";
+  return FeatureMap;
+}
+
+void TargetFeaturesEmitter::printFeatureMask(
+    raw_ostream &OS, ArrayRef<const Record *> FeatureList,
+    const FeatureMapTy &FeatureMap) {
+  std::array<uint64_t, MAX_SUBTARGET_WORDS> Mask = {};
+  for (const Record *Feature : FeatureList) {
+    unsigned Bit = FeatureMap.lookup(Feature);
+    Mask[Bit / 64] |= 1ULL << (Bit % 64);
+  }
+
+  OS << "{ { { ";
+  for (unsigned I = 0; I != Mask.size(); ++I) {
+    OS << "0x";
+    OS.write_hex(Mask[I]);
+    OS << "ULL, ";
+  }
+  OS << "} } }";
+}
+
+void TargetFeaturesEmitter::printFeatureKeyValues(
+    raw_ostream &OS, const FeatureMapTy &FeatureMap) {
+  std::vector<const Record *> FeatureList =
+      Records.getAllDerivedDefinitions("SubtargetFeature");
+
+  // Remove features with empty name.
+  llvm::erase_if(FeatureList, [](const Record *Rec) {
+    return Rec->getValueAsString("Name").empty();
+  });
+
+  if (FeatureList.empty())
+    return;
+
+  llvm::sort(FeatureList, LessRecordFieldName());
+
+  // Begin feature table.
+  OS << "// Sorted (by key) array of values for CPU features.\n"
+     << "extern const llvm::BasicSubtargetFeatureKV " << "Basic" << Target
+     << "FeatureKV[] = {\n";
+
+  for (const Record *Feature : FeatureList) {
+    StringRef Name = Feature->getName();
+    StringRef ValueName = Feature->getValueAsString("Name");
+
+    OS << "  { " << "\"" << ValueName << "\", " << Target << "::" << Name
+       << ", ";
+
+    ConstRecVec ImpliesList = Feature->getValueAsListOfDefs("Implies");
+
+    printFeatureMask(OS, ImpliesList, FeatureMap);
+
+    OS << " },\n";
+  }
+
+  // End feature table.
+  OS << "};\n";
+
+  return;
+}
+
+void TargetFeaturesEmitter::printCPUKeyValues(raw_ostream &OS,
+                                              const FeatureMapTy &FeatureMap) {
+  // Gather and sort processor information
+  std::vector<const Record *> ProcessorList =
+      Records.getAllDerivedDefinitions("Processor");
+  llvm::sort(ProcessorList, LessRecordFieldName());
+
+  // Begin processor table.
+  OS << "// Sorted (by key) array of values for CPU subtype.\n"
+     << "extern const llvm::BasicSubtargetSubTypeKV " << "Basic" << Target
+     << "SubTypeKV[] = {\n";
+
+  for (const Record *Processor : ProcessorList) {
+    StringRef Name = Processor->getValueAsString("Name");
+    ConstRecVec FeatureList = Processor->getValueAsListOfDefs("Features");
+
+    OS << " { " << "\"" << Name << "\", ";
+
+    printFeatureMask(OS, FeatureList, FeatureMap);
+    OS << " },\n";
+  }
+
+  // End processor table.
+  OS << "};\n";
+
+  return;
+}
+
+void TargetFeaturesEmitter::run(raw_ostream &OS) {
+  OS << "// Autogenerated by TargetFeatureEmitter.cpp\n\n";
+
+  OS << "\n#ifdef GET_SUBTARGETFEATURES_ENUM\n";
+  OS << "#undef GET_SUBTARGETFEATURES_ENUM\n\n";
+
+  OS << "namespace llvm {\n";
+  auto FeatureMap = enumeration(OS);
+  OS << "} // end namespace llvm\n\n";
+  OS << "#endif // GET_SUBTARGETFEATURES_ENUM\n\n";
+
+  OS << "\n#ifdef GET_SUBTARGETFEATURES_KV\n";
+  OS << "#undef GET_SUBTARGETFEATURES_KV\n\n";
+
+  OS << "namespace llvm {\n";
+  printFeatureKeyValues(OS, FeatureMap);
+  OS << "\n";
+
+  printCPUKeyValues(OS, FeatureMap);
+  OS << "\n";
+  OS << "} // end namespace llvm\n\n";
+  OS << "#endif // GET_SUBTARGETFEATURES_KV\n\n";
+}
+
+static TableGen::Emitter::OptClass<TargetFeaturesEmitter>
+    X("gen-target-features", "Generate subtarget enumerations");
diff --git a/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.h b/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.h
new file mode 100644
index 000000000000..99e4820c614c
--- /dev/null
+++ b/llvm/utils/TableGen/Basic/TargetFeaturesEmitter.h
@@ -0,0 +1,49 @@
+//===- TargetFeaturesEmitter.h- Generate CPU Target features ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines the TargetFeaturesEmitter class, which is used to export
+// CPU target features and CPU subtypes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_UTILS_TABLEGEN_BASIC_EMITTARGETFEATURE_H
+#define LLVM_UTILS_TABLEGEN_BASIC_EMITTARGETFEATURE_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/TableGen/Record.h"
+
+namespace llvm {
+/// Sorting predicate to sort record pointers by their
+/// FieldName field.
+struct LessRecordFieldFieldName {
+  bool operator()(const Record *Rec1, const Record *Rec2) const {
+    return Rec1->getValueAsString("FieldName") <
+           Rec2->getValueAsString("FieldName");
+  }
+};
+
+using FeatureMapTy = DenseMap<const Record *, unsigned>;
+
+class TargetFeaturesEmitter {
+protected:
+  const RecordKeeper &Records;
+  std::string Target;
+
+public:
+  TargetFeaturesEmitter(const RecordKeeper &R);
+  static void printFeatureMask(raw_ostream &OS,
+                               ArrayRef<const Record *> FeatureList,
+                               const FeatureMapTy &FeatureMap);
+  FeatureMapTy enumeration(raw_ostream &OS);
+  void printFeatureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
+  void printCPUKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
+  virtual void run(raw_ostream &O);
+  virtual ~TargetFeaturesEmitter() {};
+};
+} // namespace llvm
+#endif
diff --git a/llvm/utils/TableGen/SubtargetEmitter.cpp b/llvm/utils/TableGen/SubtargetEmitter.cpp
index ca008e256a70..77618b24cf11 100644
--- a/llvm/utils/TableGen/SubtargetEmitter.cpp
+++ b/llvm/utils/TableGen/SubtargetEmitter.cpp
@@ -10,12 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "Basic/TargetFeaturesEmitter.h"
 #include "Common/CodeGenHwModes.h"
 #include "Common/CodeGenSchedule.h"
 #include "Common/CodeGenTarget.h"
 #include "Common/PredicateExpander.h"
 #include "Common/Utils.h"
-#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringExtras.h"
@@ -27,9 +27,7 @@
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
-#include "llvm/TableGen/Record.h"
 #include "llvm/TableGen/TableGenBackend.h"
-#include "llvm/TargetParser/SubtargetFeature.h"
 #include <algorithm>
 #include <cassert>
 #include <cstdint>
@@ -43,18 +41,7 @@ using namespace llvm;
 
 namespace {
 
-using FeatureMapTy = DenseMap<const Record *, unsigned>;
-
-/// Sorting predicate to sort record pointers by their
-/// FieldName field.
-struct LessRecordFieldFieldName {
-  bool operator()(const Record *Rec1, const Record *Rec2) const {
-    return Rec1->getValueAsString("FieldName") <
-           Rec2->getValueAsString("FieldName");
-  }
-};
-
-class SubtargetEmitter {
+class SubtargetEmitter : TargetFeaturesEmitter {
   // Each processor has a SchedClassDesc table with an entry for each
   // SchedClass. The SchedClassDesc table indexes into a global write resource
   // table, write latency table, and read advance table.
@@ -83,11 +70,8 @@ class SubtargetEmitter {
   };
 
   CodeGenTarget TGT;
-  const RecordKeeper &Records;
   CodeGenSchedModels &SchedModels;
-  std::string Target;
 
-  FeatureMapTy enumeration(raw_ostream &OS);
   void emitSubtargetInfoMacroCalls(raw_ostream &OS);
   unsigned featureKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
   unsigned cpuKeyValues(raw_ostream &OS, const FeatureMapTy &FeatureMap);
@@ -143,73 +127,13 @@ class SubtargetEmitter {
 
 public:
   SubtargetEmitter(const RecordKeeper &R)
-      : TGT(R), Records(R), SchedModels(TGT.getSchedModels()),
-        Target(TGT.getName()) {}
+      : TargetFeaturesEmitter(R), TGT(R), SchedModels(TGT.getSchedModels()) {}
 
-  void run(raw_ostream &O);
+  void run(raw_ostream &O) override;
 };
 
 } // end anonymous namespace
 
-//
-// Enumeration - Emit the specified class as an enumeration.
-//
-FeatureMapTy SubtargetEmitter::enumeration(raw_ostream &OS) {
-  ArrayRef<const Record *> DefList =
-      Records.getAllDerivedDefinitions("SubtargetFeature");
-
-  unsigned N = DefList.size();
-  if (N == 0)
-    return FeatureMapTy();
-  if (N + 1 > MAX_SUBTARGET_FEATURES)
-    PrintFatalError(
-        "Too many subtarget features! Bump MAX_SUBTARGET_FEATURES.");
-
-  OS << "namespace " << Target << " {\n";
-
-  // Open enumeration.
-  OS << "enum {\n";
-
-  FeatureMapTy FeatureMap;
-  // For each record
-  for (unsigned I = 0; I < N; ++I) {
-    // Next record
-    const Record *Def = DefList[I];
-
-    // Get and emit name
-    OS << "  " << Def->getName() << " = " << I << ",\n";
-
-    // Save the index for this feature.
-    FeatureMap[Def] = I;
-  }
-
-  OS << "  "
-     << "NumSubtargetFeatures = " << N << "\n";
-
-  // Close enumeration and namespace
-  OS << "};\n";
-  OS << "} // end namespace " << Target << "\n";
-  return FeatureMap;
-}
-
-static void printFeatureMask(raw_ostream &OS,
-                             ArrayRef<const Record *> FeatureList,
-                             const FeatureMapTy &FeatureMap) {
-  std::array<uint64_t, MAX_SUBTARGET_WORDS> Mask = {};
-  for (const Record *Feature : FeatureList) {
-    unsigned Bit = FeatureMap.lookup(Feature);
-    Mask[Bit / 64] |= 1ULL << (Bit % 64);
-  }
-
-  OS << "{ { { ";
-  for (unsigned I = 0; I != Mask.size(); ++I) {
-    OS << "0x";
-    OS.write_hex(Mask[I]);
-    OS << "ULL, ";
-  }
-  OS << "} } }";
-}
-
 /// Emit some information about the SubtargetFeature as calls to a macro so
 /// that they can be used from C++.
 void SubtargetEmitter::emitSubtargetInfoMacroCalls(raw_ostream &OS) {
-- 
cgit v1.2.3


From f87b6625d64c4ba95cf26b249ce6bdbcb31d65c9 Mon Sep 17 00:00:00 2001
From: Abdul Raheem <abdulraheembeigh@gmail.com>
Date: Thu, 19 Jun 2025 18:54:34 +0530
Subject: [MLIR][NFC] Fixed some Typos (#144263)

-- Fixed some typos in Operation.h

Signed-off: Abdul Raheem Beigh abdulraheembeigh@gmail.com
---
 mlir/include/mlir/IR/Operation.h | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/IR/Operation.h b/mlir/include/mlir/IR/Operation.h
index 68ab1527b480..1c2c04e718bf 100644
--- a/mlir/include/mlir/IR/Operation.h
+++ b/mlir/include/mlir/IR/Operation.h
@@ -30,7 +30,7 @@ enum class OpProperties : char {};
 
 /// Operation is the basic unit of execution within MLIR.
 ///
-/// The following documentation are recommended to understand this class:
+/// The following documentations are recommended to understand this class:
 /// - https://mlir.llvm.org/docs/LangRef/#operations
 /// - https://mlir.llvm.org/docs/Tutorials/UnderstandingTheIRStructure/
 ///
@@ -66,14 +66,14 @@ enum class OpProperties : char {};
 /// tail allocated with the operation class itself, but can be dynamically moved
 /// out-of-line in a dynamic allocation as needed.
 ///
-/// An Operation may contain optionally one or multiple Regions, stored in a
+/// An Operation may optionally contain one or multiple Regions, stored in a
 /// tail allocated array. Each `Region` is a list of Blocks. Each `Block` is
 /// itself a list of Operations. This structure is effectively forming a tree.
 ///
 /// Some operations like branches also refer to other Block, in which case they
 /// would have an array of `BlockOperand`.
 ///
-/// An Operation may contain optionally a "Properties" object: this is a
+/// An Operation may optionally contain a "Properties" object: this is a
 /// pre-defined C++ object with a fixed size. This object is owned by the
 /// operation and deleted with the operation. It can be converted to an
 /// Attribute on demand, or loaded from an Attribute.
@@ -286,7 +286,7 @@ public:
   void destroy();
 
   /// This drops all operand uses from this operation, which is an essential
-  /// step in breaking cyclic dependences between references when they are to
+  /// step in breaking cyclic dependencies between references when they are to
   /// be deleted.
   void dropAllReferences();
 
@@ -448,11 +448,11 @@ public:
   /// to use Properties instead.
   void setInherentAttr(StringAttr name, Attribute value);
 
-  /// Access a discardable attribute by name, returns an null Attribute if the
+  /// Access a discardable attribute by name, returns a null Attribute if the
   /// discardable attribute does not exist.
   Attribute getDiscardableAttr(StringRef name) { return attrs.get(name); }
 
-  /// Access a discardable attribute by name, returns an null Attribute if the
+  /// Access a discardable attribute by name, returns a null Attribute if the
   /// discardable attribute does not exist.
   Attribute getDiscardableAttr(StringAttr name) { return attrs.get(name); }
 
@@ -515,7 +515,7 @@ public:
   DictionaryAttr getAttrDictionary();
 
   /// Set the attributes from a dictionary on this operation.
-  /// These methods are expensive: if the dictionnary only contains discardable
+  /// These methods are expensive: if the dictionary only contains discardable
   /// attributes, `setDiscardableAttrs` is more efficient.
   void setAttrs(DictionaryAttr newAttrs);
   void setAttrs(ArrayRef<NamedAttribute> newAttrs);
@@ -529,7 +529,7 @@ public:
   }
 
   /// Return the specified attribute if present, null otherwise.
-  /// These methods are expensive: if the dictionnary only contains discardable
+  /// These methods are expensive: if the dictionary only contains discardable
   /// attributes, `getDiscardableAttr` is more efficient.
   Attribute getAttr(StringAttr name) {
     if (getPropertiesStorageSize()) {
@@ -950,7 +950,7 @@ private:
   /// operation.
   static constexpr unsigned kOrderStride = 5;
 
-  /// Update the order index of this operation of this operation if necessary,
+  /// Update the order index of this operation if necessary,
   /// potentially recomputing the order of the parent block.
   void updateOrderIfNecessary();
 
-- 
cgit v1.2.3


From e75e2485f2e5e627d0bdf0306df4672f69ddd6eb Mon Sep 17 00:00:00 2001
From: Brox Chen <guochen2@amd.com>
Date: Thu, 19 Jun 2025 09:26:45 -0400
Subject: [AMDGPU][True16][Codegen] keep srcmod/clamp/omod from v_s_xxx_f16
 when moved to VALU (#144781)

https://github.com/llvm/llvm-project/pull/141152 causes an issue in
v_s_xxx_f16 lowering in both true16/fake16 flow.

V_S_XXX_F16 are special insts which has scalar input/output but in VALU
VOP3 format. Need to keep the srcmod/clamp/omod when lower it to its
corresponding VALU inst with vector input/output.
---
 llvm/lib/Target/AMDGPU/SIInstrInfo.cpp             |  9 +--
 .../AMDGPU/fix-sgpr-copies-f16-gfx12-fake16.mir    | 78 ++++++++++++++++++++++
 .../AMDGPU/fix-sgpr-copies-f16-gfx12-true16.mir    | 78 ++++++++++++++++++++++
 .../move-to-valu-pseudo-scalar-trans-f16-fake16.ll | 10 +--
 .../move-to-valu-pseudo-scalar-trans-f16-true16.ll | 10 +--
 5 files changed, 171 insertions(+), 14 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-fake16.mir
 create mode 100644 llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-true16.mir

diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 2ebf8b99e9d7..a538ec9df6f0 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -7744,11 +7744,12 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
                                                     ? &AMDGPU::VGPR_16RegClass
                                                     : &AMDGPU::VGPR_32RegClass);
     auto NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
-                        .addImm(0) // src0_modifiers
+                        .add(Inst.getOperand(1)) // src0_modifiers
                         .add(Inst.getOperand(2))
-                        .addImm(0)  // clamp
-                        .addImm(0); // omod
-    if (ST.useRealTrue16Insts())
+                        .add(Inst.getOperand(3)) // clamp
+                        .add(Inst.getOperand(4)) // omod
+                        .setMIFlags(Inst.getFlags());
+    if (AMDGPU::hasNamedOperand(NewOpcode, AMDGPU::OpName::op_sel))
       NewInstr.addImm(0); // opsel0
     MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
     legalizeOperandsVALUt16(*NewInstr, MRI);
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-fake16.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-fake16.mir
new file mode 100644
index 000000000000..1ec7249476ec
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-fake16.mir
@@ -0,0 +1,78 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+
+---
+name:            v_s_exp_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_exp_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_EXP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_EXP_F16_fake16_e64 1, [[V_CVT_F32_U32_e64_]], 1, 1, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_EXP_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_log_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_log_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_LOG_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_LOG_F16_fake16_e64 1, [[V_CVT_F32_U32_e64_]], 1, 1, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_LOG_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_rcp_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_rcp_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_RCP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RCP_F16_fake16_e64 1, [[V_CVT_F32_U32_e64_]], 1, 1, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_RCP_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_rsq_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_rsq_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_RSQ_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RSQ_F16_fake16_e64 1, [[V_CVT_F32_U32_e64_]], 1, 1, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_RSQ_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_sqrt_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_sqrt_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_SQRT_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_SQRT_F16_fake16_e64 1, [[V_CVT_F32_U32_e64_]], 1, 1, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_SQRT_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-true16.mir b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-true16.mir
new file mode 100644
index 000000000000..5194d2529597
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fix-sgpr-copies-f16-gfx12-true16.mir
@@ -0,0 +1,78 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -run-pass=si-fix-sgpr-copies -verify-machineinstrs -o - %s | FileCheck --check-prefixes=GCN %s
+
+---
+name:            v_s_exp_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_exp_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_EXP_F16_t16_e64 1, [[V_CVT_F32_U32_e64_]].lo16, 1, 1, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_EXP_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_log_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_log_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LOG_F16_t16_e64 1, [[V_CVT_F32_U32_e64_]].lo16, 1, 1, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_LOG_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_rcp_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_rcp_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RCP_F16_t16_e64 1, [[V_CVT_F32_U32_e64_]].lo16, 1, 1, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_RCP_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_rsq_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_rsq_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RSQ_F16_t16_e64 1, [[V_CVT_F32_U32_e64_]].lo16, 1, 1, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_RSQ_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
+---
+name:            v_s_sqrt_f16
+body:             |
+  bb.0.entry:
+    ; GCN-LABEL: name: v_s_sqrt_f16
+    ; GCN: [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_CVT_F32_U32_e64_:%[0-9]+]]:vgpr_32 = V_CVT_F32_U32_e64 [[DEF]], 0, 0, implicit $mode, implicit $exec
+    ; GCN-NEXT: [[DEF1:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
+    ; GCN-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_SQRT_F16_t16_e64 1, [[V_CVT_F32_U32_e64_]].lo16, 1, 1, 0, implicit $mode, implicit $exec
+    %0:vgpr_32 = IMPLICIT_DEF
+    %1:vgpr_32 = V_CVT_F32_U32_e64 %0:vgpr_32, 0, 0, implicit $mode, implicit $exec
+    %2:sreg_32 = COPY %1:vgpr_32
+    %3:sreg_32_xexec = V_S_SQRT_F16_e64 1, %2:sreg_32, 1, 1, implicit $mode, implicit $exec
+...
+
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll
index a6819359561b..2870af19f94c 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-fake16.ll
@@ -11,7 +11,7 @@ define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_EXP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_EXP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_EXP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_EXP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_EXP_F16_fake16_e64_]]
   ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
@@ -32,7 +32,7 @@ define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_LOG_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_LOG_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_LOG_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_LOG_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_LOG_F16_fake16_e64_]]
   ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
@@ -53,7 +53,7 @@ define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_RCP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RCP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_RCP_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RCP_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RCP_F16_fake16_e64_]]
   ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
@@ -74,7 +74,7 @@ define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_RSQ_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_RSQ_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_RSQ_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_RSQ_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_RSQ_F16_fake16_e64_]]
   ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
@@ -95,7 +95,7 @@ define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
   ; CHECK-NEXT:   [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:sreg_32 = IMPLICIT_DEF
-  ; CHECK-NEXT:   [[V_SQRT_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = V_SQRT_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_SQRT_F16_fake16_e64_:%[0-9]+]]:vgpr_32 = nofpexcept V_SQRT_F16_fake16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]], 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:sreg_32_xexec = IMPLICIT_DEF
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY [[V_SQRT_F16_fake16_e64_]]
   ; CHECK-NEXT:   GLOBAL_STORE_SHORT_SADDR [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
index b1b5b6b62296..c93eb1d9ef14 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
-  ; CHECK-NEXT:   [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
@@ -35,7 +35,7 @@ define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
-  ; CHECK-NEXT:   [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
@@ -58,7 +58,7 @@ define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
-  ; CHECK-NEXT:   [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
@@ -81,7 +81,7 @@ define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
-  ; CHECK-NEXT:   [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
@@ -104,7 +104,7 @@ define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
   ; CHECK-NEXT:   [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
   ; CHECK-NEXT:   [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
-  ; CHECK-NEXT:   [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+  ; CHECK-NEXT:   [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
   ; CHECK-NEXT:   [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
   ; CHECK-NEXT:   [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
   ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
-- 
cgit v1.2.3


From 278ece7c80d36bb1074fa53e655a5ca8f31145dd Mon Sep 17 00:00:00 2001
From: Abhishek Kaushik <abhishek.kaushik@intel.com>
Date: Thu, 19 Jun 2025 06:29:05 -0700
Subject: [InstCombine][NFC] Pre-commit tests for #125935 (#144111)

Pre-commit tests for #125935

---------

Co-authored-by: Simon Pilgrim <llvm-dev@redking.me.uk>
---
 .../Transforms/InstCombine/bitcast-known-bits.ll   | 134 +++++++++++++++++++++
 1 file changed, 134 insertions(+)
 create mode 100644 llvm/test/Transforms/InstCombine/bitcast-known-bits.ll

diff --git a/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll b/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll
new file mode 100644
index 000000000000..3e47e775e3a2
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; PR125228
+
+define <16 x i8> @knownbits_bitcast_masked_shift(<16 x i8> %arg1, <16 x i8> %arg2)  {
+; CHECK-LABEL: define <16 x i8> @knownbits_bitcast_masked_shift(
+; CHECK-SAME: <16 x i8> [[ARG1:%.*]], <16 x i8> [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <16 x i8> [[ARG1]], splat (i8 3)
+; CHECK-NEXT:    [[AND3:%.*]] = and <16 x i8> [[ARG2]], splat (i8 48)
+; CHECK-NEXT:    [[OR:%.*]] = or disjoint <16 x i8> [[AND3]], [[AND]]
+; CHECK-NEXT:    [[BITCAST4:%.*]] = bitcast <16 x i8> [[OR]] to <8 x i16>
+; CHECK-NEXT:    [[SHL5:%.*]] = shl nuw <8 x i16> [[BITCAST4]], splat (i16 2)
+; CHECK-NEXT:    [[BITCAST6:%.*]] = bitcast <8 x i16> [[SHL5]] to <16 x i8>
+; CHECK-NEXT:    [[AND7:%.*]] = and <16 x i8> [[BITCAST6]], splat (i8 -52)
+; CHECK-NEXT:    ret <16 x i8> [[AND7]]
+;
+  %and = and <16 x i8> %arg1, splat (i8 3)
+  %and3 = and <16 x i8> %arg2, splat (i8 48)
+  %or = or disjoint <16 x i8> %and3, %and
+  %bitcast4 = bitcast <16 x i8> %or to <8 x i16>
+  %shl5 = shl nuw <8 x i16> %bitcast4, splat (i16 2)
+  %bitcast6 = bitcast <8 x i16> %shl5 to <16 x i8>
+  %and7 = and <16 x i8> %bitcast6, splat (i8 -52)
+  ret <16 x i8> %and7
+}
+
+define <16 x i8> @knownbits_shuffle_masked_nibble_shift(<16 x i8> %arg)  {
+; CHECK-LABEL: define <16 x i8> @knownbits_shuffle_masked_nibble_shift(
+; CHECK-SAME: <16 x i8> [[ARG:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <16 x i8> [[ARG]], splat (i8 15)
+; CHECK-NEXT:    [[SHUFFLEVECTOR:%.*]] = shufflevector <16 x i8> [[AND]], <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <16 x i8> [[SHUFFLEVECTOR]] to <8 x i16>
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <8 x i16> [[BITCAST1]], splat (i16 4)
+; CHECK-NEXT:    [[BITCAST2:%.*]] = bitcast <8 x i16> [[SHL]] to <16 x i8>
+; CHECK-NEXT:    [[AND3:%.*]] = and <16 x i8> [[BITCAST2]], splat (i8 -16)
+; CHECK-NEXT:    ret <16 x i8> [[AND3]]
+;
+  %and = and <16 x i8> %arg, splat (i8 15)
+  %shufflevector = shufflevector <16 x i8> %and, <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
+  %bitcast1 = bitcast <16 x i8> %shufflevector to <8 x i16>
+  %shl = shl nuw <8 x i16> %bitcast1, splat (i16 4)
+  %bitcast2 = bitcast <8 x i16> %shl to <16 x i8>
+  %and3 = and <16 x i8> %bitcast2, splat (i8 -16)
+  ret <16 x i8> %and3
+}
+
+define <16 x i8> @knownbits_reverse_shuffle_masked_shift(<16 x i8> %arg)  {
+; CHECK-LABEL: define <16 x i8> @knownbits_reverse_shuffle_masked_shift(
+; CHECK-SAME: <16 x i8> [[ARG:%.*]]) {
+; CHECK-NEXT:    [[AND:%.*]] = and <16 x i8> [[ARG]], splat (i8 15)
+; CHECK-NEXT:    [[SHUFFLEVECTOR:%.*]] = shufflevector <16 x i8> [[AND]], <16 x i8> poison, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <16 x i8> [[SHUFFLEVECTOR]] to <8 x i16>
+; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <8 x i16> [[BITCAST1]], splat (i16 4)
+; CHECK-NEXT:    [[BITCAST2:%.*]] = bitcast <8 x i16> [[SHL]] to <16 x i8>
+; CHECK-NEXT:    [[AND3:%.*]] = and <16 x i8> [[BITCAST2]], splat (i8 -16)
+; CHECK-NEXT:    ret <16 x i8> [[AND3]]
+;
+  %and = and <16 x i8> %arg, splat (i8 15)
+  %shufflevector = shufflevector <16 x i8> %and, <16 x i8> poison, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
+  %bitcast1 = bitcast <16 x i8> %shufflevector to <8 x i16>
+  %shl = shl nuw <8 x i16> %bitcast1, splat (i16 4)
+  %bitcast2 = bitcast <8 x i16> %shl to <16 x i8>
+  %and3 = and <16 x i8> %bitcast2, splat (i8 -16)
+  ret <16 x i8> %and3
+}
+
+define <16 x i8> @knownbits_extract_bit(<8 x i16> %arg)  {
+; CHECK-LABEL: define <16 x i8> @knownbits_extract_bit(
+; CHECK-SAME: <8 x i16> [[ARG:%.*]]) {
+; CHECK-NEXT:    [[LSHR:%.*]] = lshr <8 x i16> [[ARG]], splat (i16 15)
+; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <8 x i16> [[LSHR]] to <16 x i8>
+; CHECK-NEXT:    [[AND:%.*]] = and <16 x i8> [[BITCAST1]], splat (i8 1)
+; CHECK-NEXT:    ret <16 x i8> [[AND]]
+;
+  %lshr = lshr <8 x i16> %arg, splat (i16 15)
+  %bitcast1 = bitcast <8 x i16> %lshr to <16 x i8>
+  %and = and <16 x i8> %bitcast1, splat (i8 1)
+  ret <16 x i8> %and
+}
+
+define { i32, i1 } @knownbits_popcount_add_with_overflow(<2 x i64> %arg1, <2 x i64> %arg2)  {
+; CHECK-LABEL: define { i32, i1 } @knownbits_popcount_add_with_overflow(
+; CHECK-SAME: <2 x i64> [[ARG1:%.*]], <2 x i64> [[ARG2:%.*]]) {
+; CHECK-NEXT:    [[CALL:%.*]] = tail call range(i64 0, 65) <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[ARG1]])
+; CHECK-NEXT:    [[BITCAST5:%.*]] = bitcast <2 x i64> [[CALL]] to <4 x i32>
+; CHECK-NEXT:    [[EXTRACTELEMENT:%.*]] = extractelement <4 x i32> [[BITCAST5]], i64 0
+; CHECK-NEXT:    [[CALL9:%.*]] = tail call range(i64 0, 65) <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[ARG2]])
+; CHECK-NEXT:    [[BITCAST10:%.*]] = bitcast <2 x i64> [[CALL9]] to <4 x i32>
+; CHECK-NEXT:    [[EXTRACTELEMENT11:%.*]] = extractelement <4 x i32> [[BITCAST10]], i64 0
+; CHECK-NEXT:    [[TMP1:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[EXTRACTELEMENT]], i32 [[EXTRACTELEMENT11]])
+; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
+;
+  %call = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %arg1)
+  %bitcast5 = bitcast <2 x i64> %call to <4 x i32>
+  %extractelement = extractelement <4 x i32> %bitcast5, i64 0
+  %call9 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %arg2)
+  %bitcast10 = bitcast <2 x i64> %call9 to <4 x i32>
+  %extractelement11 = extractelement <4 x i32> %bitcast10, i64 0
+  %call12 = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %extractelement, i32 %extractelement11)
+  ret { i32, i1 } %call12
+}
+
+define <16 x i8> @knownbits_shuffle_add_shift_v32i8(<16 x i8> %arg1, <8 x i16> %arg2, <8 x i16> %arg3)  {
+; CHECK-LABEL: define <16 x i8> @knownbits_shuffle_add_shift_v32i8(
+; CHECK-SAME: <16 x i8> [[ARG1:%.*]], <8 x i16> [[ARG2:%.*]], <8 x i16> [[ARG3:%.*]]) {
+; CHECK-NEXT:    [[SHL6:%.*]] = shl <8 x i16> [[ARG2]], splat (i16 8)
+; CHECK-NEXT:    [[BITCAST7:%.*]] = bitcast <8 x i16> [[SHL6]] to <16 x i8>
+; CHECK-NEXT:    [[SHL10:%.*]] = shl <8 x i16> [[ARG3]], splat (i16 8)
+; CHECK-NEXT:    [[BITCAST11:%.*]] = bitcast <8 x i16> [[SHL10]] to <16 x i8>
+; CHECK-NEXT:    [[ADD12:%.*]] = add <16 x i8> [[BITCAST11]], [[BITCAST7]]
+; CHECK-NEXT:    [[ADD14:%.*]] = add <16 x i8> [[ADD12]], [[ARG1]]
+; CHECK-NEXT:    [[BITCAST14:%.*]] = bitcast <16 x i8> [[ADD12]] to <8 x i16>
+; CHECK-NEXT:    [[SHL15:%.*]] = shl <8 x i16> [[BITCAST14]], splat (i16 8)
+; CHECK-NEXT:    [[BITCAST16:%.*]] = bitcast <8 x i16> [[SHL15]] to <16 x i8>
+; CHECK-NEXT:    [[ADD13:%.*]] = add <16 x i8> [[ADD14]], [[BITCAST16]]
+; CHECK-NEXT:    ret <16 x i8> [[ADD13]]
+;
+  %shl6 = shl <8 x i16> %arg2, splat (i16 8)
+  %bitcast7 = bitcast <8 x i16> %shl6 to <16 x i8>
+  %shl10 = shl <8 x i16> %arg3, splat (i16 8)
+  %bitcast11 = bitcast <8 x i16> %shl10 to <16 x i8>
+  %add12 = add <16 x i8> %bitcast11, %bitcast7
+  %add13 = add <16 x i8> %add12, %arg1
+  %bitcast14 = bitcast <16 x i8> %add12 to <8 x i16>
+  %shl15 = shl <8 x i16> %bitcast14, splat (i16 8)
+  %bitcast16 = bitcast <8 x i16> %shl15 to <16 x i8>
+  %add17 = add <16 x i8> %add13, %bitcast16
+  ret <16 x i8> %add17
+}
+
+declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
+
+declare { i32, i1 } @llvm.uadd.with.overflow.i32(i32, i32)
-- 
cgit v1.2.3


From a4e4527c4b44be9a88168c0a4758de58fd1a770d Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Thu, 19 Jun 2025 15:39:06 +0200
Subject: [mlir][Transforms] Fix replaceUsesOfBlockArgument API (#144706)

Before this PR, users had to pass the "old" block argument when
replacing the uses of a block argument in a newly converted block. Users
can now pass the actual block argument that should be replaced.

Note for LLVM integration: Make sure to pass the current block argument
instead of the old one.
---
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp   | 2 +-
 mlir/lib/Transforms/Utils/DialectConversion.cpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 328c605add65..c6bcb593eaad 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -295,7 +295,7 @@ static void restoreByValRefArgumentType(
         cast<TypeAttr>(byValRefAttr->getValue()).getValue());
 
     auto valueArg = rewriter.create<LLVM::LoadOp>(arg.getLoc(), resTy, arg);
-    rewriter.replaceUsesOfBlockArgument(oldArg, valueArg);
+    rewriter.replaceUsesOfBlockArgument(arg, valueArg);
   }
 }
 
diff --git a/mlir/lib/Transforms/Utils/DialectConversion.cpp b/mlir/lib/Transforms/Utils/DialectConversion.cpp
index c4b85ec4f67d..3b669f51a615 100644
--- a/mlir/lib/Transforms/Utils/DialectConversion.cpp
+++ b/mlir/lib/Transforms/Utils/DialectConversion.cpp
@@ -1743,7 +1743,7 @@ void ConversionPatternRewriter::replaceUsesOfBlockArgument(BlockArgument from,
   });
   impl->appendRewrite<ReplaceBlockArgRewrite>(from.getOwner(), from,
                                               impl->currentTypeConverter);
-  impl->mapping.map(impl->mapping.lookupOrDefault(from), to);
+  impl->mapping.map(from, to);
 }
 
 Value ConversionPatternRewriter::getRemappedValue(Value key) {
-- 
cgit v1.2.3


From 802fa92aee3565768887615108aa3e924d4e0fc7 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 19 Jun 2025 16:17:54 +0200
Subject: [PredicateInfo] Avoid duplicate hash lookup (NFC)

Use try_emplace to either look up the existing entry or insert it.
---
 llvm/lib/Transforms/Utils/PredicateInfo.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index d4b2bcf40fd8..4c87babbfb6f 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -728,16 +728,12 @@ void PredicateInfoBuilder::renameUses(SmallVectorImpl<Value *> &OpsToRename) {
 
 PredicateInfoBuilder::ValueInfo &
 PredicateInfoBuilder::getOrCreateValueInfo(Value *Operand) {
-  auto OIN = ValueInfoNums.find(Operand);
-  if (OIN == ValueInfoNums.end()) {
-    // This will grow it
+  auto Res = ValueInfoNums.try_emplace(Operand, ValueInfos.size());
+  if (Res.second) {
+    // Allocate space for new ValueInfo.
     ValueInfos.resize(ValueInfos.size() + 1);
-    // This will use the new size and give us a 0 based number of the info
-    auto InsertResult = ValueInfoNums.insert({Operand, ValueInfos.size() - 1});
-    assert(InsertResult.second && "Value info number already existed?");
-    return ValueInfos[InsertResult.first->second];
   }
-  return ValueInfos[OIN->second];
+  return ValueInfos[Res.first->second];
 }
 
 const PredicateInfoBuilder::ValueInfo &
-- 
cgit v1.2.3


From 7e8f1f5f72753a1d9d3ae2810da52b82a787600b Mon Sep 17 00:00:00 2001
From: Siu Chi Chan <siuchi.chan@amd.com>
Date: Thu, 19 Jun 2025 14:23:23 +0000
Subject: [HIP] Remove dots in HIP runtime path (#143792)

Remove the dots in the HIP path before passing to the rpath flag
---
 clang/lib/Driver/ToolChains/Linux.cpp        | 8 +++++---
 clang/test/Driver/hip-runtime-libs-linux.hip | 6 ++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/clang/lib/Driver/ToolChains/Linux.cpp b/clang/lib/Driver/ToolChains/Linux.cpp
index 151b2bfced81..8ac8d4eb9181 100644
--- a/clang/lib/Driver/ToolChains/Linux.cpp
+++ b/clang/lib/Driver/ToolChains/Linux.cpp
@@ -743,9 +743,11 @@ void Linux::AddHIPRuntimeLibArgs(const ArgList &Args,
       Args.MakeArgString(StringRef("-L") + RocmInstallation->getLibPath()));
 
   if (Args.hasFlag(options::OPT_frtlib_add_rpath,
-                   options::OPT_fno_rtlib_add_rpath, false))
-    CmdArgs.append(
-        {"-rpath", Args.MakeArgString(RocmInstallation->getLibPath())});
+                   options::OPT_fno_rtlib_add_rpath, false)) {
+    SmallString<0> p = RocmInstallation->getLibPath();
+    llvm::sys::path::remove_dots(p, true);
+    CmdArgs.append({"-rpath", Args.MakeArgString(p)});
+  }
 
   CmdArgs.push_back("-lamdhip64");
 }
diff --git a/clang/test/Driver/hip-runtime-libs-linux.hip b/clang/test/Driver/hip-runtime-libs-linux.hip
index a4cd2733114b..eda87d0aa4b6 100644
--- a/clang/test/Driver/hip-runtime-libs-linux.hip
+++ b/clang/test/Driver/hip-runtime-libs-linux.hip
@@ -20,6 +20,11 @@
 // RUN:   --rocm-path=%S/Inputs/rocm %t.o -frtlib-add-rpath 2>&1 \
 // RUN:   | FileCheck -check-prefixes=ROCM-RPATH %s
 
+// Test that a canonical HIP runtime path is passed to the -rpath flag
+// RUN: %clang -### --hip-link --target=x86_64-linux-gnu \
+// RUN:   --rocm-path=%S/Inputs/rocm/./bin/../include/../ %t.o -frtlib-add-rpath 2>&1 \
+// RUN:   | FileCheck -check-prefixes=ROCM-RPATH-CANONICAL %s
+
 // Test detecting latest /opt/rocm-{release} directory.
 // RUN: rm -rf %t && mkdir -p %t/opt
 // RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.9.0-1234
@@ -55,6 +60,7 @@
 
 // ROCM-PATH: "-L[[HIPRT:.*/Inputs/rocm/lib]]" "-lamdhip64"
 // ROCM-RPATH: "-L[[HIPRT:.*/Inputs/rocm/lib]]" "-rpath" "[[HIPRT]]" "-lamdhip64"
+// ROCM-RPATH-CANONICAL: "-rpath" "{{.*/rocm/lib}}" "-lamdhip64"
 // ROCM-REL: "-L[[HIPRT:.*/opt/rocm-3.10.0/lib]]" "-lamdhip64"
 // NOHIPRT-NOT: "-L{{.*/Inputs/rocm/lib}}"
 // NOHIPRT-NOT: "-rpath" "{{.*/Inputs/rocm/lib}}"
-- 
cgit v1.2.3


From 4c6f398b866030c17fd94dcdca04f4df03c5214c Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Thu, 19 Jun 2025 09:23:57 -0500
Subject: [Clang] Add standalone AMDGPU SPIR-V toolchain (#144576)

Summary:
The AMDGPU toolchain uses a different set of tools than the standard
SPIR-V toolchain. The linker wrapper prefers to invoke a linker via a
clang toolchain. To make that work we introduce
`--target=spirv64-amd-amdhsa` so that it creates the linking phases that
HIP prefers. Additionally, this can be used to make LLVM-IR / SPIR-V
from C/C++ that can be linked with the HIP output.
---
 clang/lib/Driver/Driver.cpp             | 16 +++++++++++-----
 clang/lib/Driver/ToolChains/HIPAMD.cpp  | 12 ++++++++++++
 clang/lib/Driver/ToolChains/HIPAMD.h    |  9 +++++++++
 clang/test/Driver/spirv-amd-toolchain.c | 19 +++++++++++++++++++
 4 files changed, 51 insertions(+), 5 deletions(-)
 create mode 100644 clang/test/Driver/spirv-amd-toolchain.c

diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 2f86b6633df1..061d60c62903 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -6857,11 +6857,17 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
       TC = std::make_unique<toolchains::NVPTXToolChain>(*this, Target, Args);
       break;
     case llvm::Triple::AMDHSA: {
-      bool DL =
-          usesInput(Args, types::isOpenCL) || usesInput(Args, types::isLLVMIR);
-      TC = DL ? std::make_unique<toolchains::ROCMToolChain>(*this, Target, Args)
-              : std::make_unique<toolchains::AMDGPUToolChain>(*this, Target,
-                                                              Args);
+      if (Target.getArch() == llvm::Triple::spirv64) {
+        TC = std::make_unique<toolchains::SPIRVAMDToolChain>(*this, Target,
+                                                             Args);
+      } else {
+        bool DL = usesInput(Args, types::isOpenCL) ||
+                  usesInput(Args, types::isLLVMIR);
+        TC = DL ? std::make_unique<toolchains::ROCMToolChain>(*this, Target,
+                                                              Args)
+                : std::make_unique<toolchains::AMDGPUToolChain>(*this, Target,
+                                                                Args);
+      }
       break;
     }
     case llvm::Triple::AMDPAL:
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index a8f2b09b1b20..74ac8306e7cc 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -417,3 +417,15 @@ void HIPAMDToolChain::checkTargetID(
     getDriver().Diag(clang::diag::err_drv_bad_target_id)
         << *PTID.OptionalTargetID;
 }
+
+SPIRVAMDToolChain::SPIRVAMDToolChain(const Driver &D,
+                                     const llvm::Triple &Triple,
+                                     const ArgList &Args)
+    : ROCMToolChain(D, Triple, Args) {
+  getProgramPaths().push_back(getDriver().Dir);
+}
+
+Tool *SPIRVAMDToolChain::buildLinker() const {
+  assert(getTriple().getArch() == llvm::Triple::spirv64);
+  return new tools::AMDGCN::Linker(*this);
+}
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.h b/clang/lib/Driver/ToolChains/HIPAMD.h
index c31894e22c5c..3630b11cd8b1 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.h
+++ b/clang/lib/Driver/ToolChains/HIPAMD.h
@@ -97,6 +97,15 @@ protected:
   Tool *buildLinker() const override;
 };
 
+class LLVM_LIBRARY_VISIBILITY SPIRVAMDToolChain final : public ROCMToolChain {
+public:
+  SPIRVAMDToolChain(const Driver &D, const llvm::Triple &Triple,
+                    const llvm::opt::ArgList &Args);
+
+protected:
+  Tool *buildLinker() const override;
+};
+
 } // end namespace toolchains
 } // end namespace driver
 } // end namespace clang
diff --git a/clang/test/Driver/spirv-amd-toolchain.c b/clang/test/Driver/spirv-amd-toolchain.c
new file mode 100644
index 000000000000..c9417400a937
--- /dev/null
+++ b/clang/test/Driver/spirv-amd-toolchain.c
@@ -0,0 +1,19 @@
+// RUN: %clang -### -ccc-print-phases --target=spirv64-amd-amdhsa %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=PHASES
+// PHASES: 0: input, "[[INPUT:.+]]", c
+// PHASES: 1: preprocessor, {0}, cpp-output
+// PHASES: 2: compiler, {1}, ir
+// PHASES: 3: backend, {2}, assembler
+// PHASES: 4: assembler, {3}, object
+// PHASES: 5: linker, {4}, image
+
+// RUN: %clang -### -ccc-print-bindings --target=spirv64-amd-amdhsa %s 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=BINDINGS
+// BINDINGS: # "spirv64-amd-amdhsa" - "clang", inputs: ["[[INPUT:.+]]"], output: "[[OUTPUT:.+]]"
+// BINDINGS: # "spirv64-amd-amdhsa" - "AMDGCN::Linker", inputs: ["[[OUTPUT]]"], output: "a.out"
+
+// RUN: %clang -### --target=spirv64-amd-amdhsa %s -nogpulib -nogpuinc 2>&1 \
+// RUN:   | FileCheck %s --check-prefix=INVOCATION
+// INVOCATION: "-cc1" "-triple" "spirv64-amd-amdhsa" {{.*}} "-o" "[[OUTPUT:.+]]" "-x" "c"
+// INVOCATION: "{{.*}}llvm-link" "-o" "a.out" "[[OUTPUT]]"
+// INVOCATION: "{{.*}}llvm-spirv" "--spirv-max-version=1.6" "--spirv-ext=+all" "--spirv-allow-unknown-intrinsics" "--spirv-lower-const-expr" "--spirv-preserve-auxdata" "--spirv-debug-info-version=nonsemantic-shader-200" "a.out" "-o" "a.out"
-- 
cgit v1.2.3


From 19360e62d0d1a1dabf9f01736927ab8f1b72c7df Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Thu, 19 Jun 2025 10:20:36 -0400
Subject: [gn build] port bf79d4819ede (ppc -gen-target-features)

---
 llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn | 8 ++++++++
 llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn      | 1 +
 2 files changed, 9 insertions(+)

diff --git a/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn b/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
index 455a8265fce8..c3cdf46ccfb9 100644
--- a/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/include/llvm/TargetParser/BUILD.gn
@@ -14,6 +14,13 @@ tablegen("AArch64TargetParserDef") {
   tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
 }
 
+tablegen("PPCGenTargetFeatures") {
+  visibility = [ ":gen" ]
+  args = [ "-gen-target-features" ]
+  td_file = "//llvm/lib/Target/PowerPC/PPC.td"
+  tblgen_target = "//llvm/utils/TableGen:llvm-min-tblgen"
+}
+
 tablegen("RISCVTargetParserDef") {
   visibility = [ ":gen" ]
   args = [ "-gen-riscv-target-def" ]
@@ -25,6 +32,7 @@ group("gen") {
   deps = [
     ":AArch64TargetParserDef",
     ":ARMTargetParserDef",
+    ":PPCGenTargetFeatures",
     ":RISCVTargetParserDef",
   ]
 }
diff --git a/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
index ef6d6e44b6f8..5838118f2711 100644
--- a/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/TableGen/Basic/BUILD.gn
@@ -12,6 +12,7 @@ source_set("Basic") {
     "RISCVTargetDefEmitter.cpp",
     "SDNodeProperties.cpp",
     "TableGen.cpp",
+    "TargetFeaturesEmitter.cpp",
     "VTEmitter.cpp",
   ]
 }
-- 
cgit v1.2.3


From 89efae916a5de0387710b7dc06938423817e1503 Mon Sep 17 00:00:00 2001
From: Jack Styles <jack.styles@arm.com>
Date: Thu, 19 Jun 2025 15:32:27 +0100
Subject: [Flang][OpenMP] Update default MapType for Map Clauses and OpenMP 5.2
 (#144715)

In OpenMP 5.2, the `target enter data` and `target exit data` constructs
now have default map types if the user does not define them in the Map
clause. For `target enter data`, this is `to` and `target exit data`
this is `from`. This behaviour is now enabled when OpenMP 5.2 or greater
is used when compiling. To enable this, the default value is now set in
the `processMap` clause, with any previous behaviour being maintained
for either older versions of OpenMP or other directives.

See also #110008
---
 flang/lib/Lower/OpenMP/ClauseProcessor.cpp         | 17 ++++++++++----
 flang/lib/Lower/OpenMP/ClauseProcessor.h           |  1 +
 flang/lib/Lower/OpenMP/Clauses.cpp                 |  2 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp                  |  5 ++--
 .../OpenMP/target-enter-data-default-openmp52.f90  | 27 ++++++++++++++++++++++
 5 files changed, 45 insertions(+), 7 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90

diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
index bc8fc14bcaeb..7bea427099a2 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.cpp
@@ -1269,7 +1269,7 @@ void ClauseProcessor::processMapObjects(
 
 bool ClauseProcessor::processMap(
     mlir::Location currentLocation, lower::StatementContext &stmtCtx,
-    mlir::omp::MapClauseOps &result,
+    mlir::omp::MapClauseOps &result, llvm::omp::Directive directive,
     llvm::SmallVectorImpl<const semantics::Symbol *> *mapSyms) const {
   // We always require tracking of symbols, even if the caller does not,
   // so we create an optionally used local set of symbols when the mapSyms
@@ -1287,9 +1287,18 @@ bool ClauseProcessor::processMap(
     llvm::omp::OpenMPOffloadMappingFlags mapTypeBits =
         llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_NONE;
     std::string mapperIdName = "__implicit_mapper";
-    // If the map type is specified, then process it else Tofrom is the
-    // default.
-    Map::MapType type = mapType.value_or(Map::MapType::Tofrom);
+    // If the map type is specified, then process it else set the appropriate
+    // default value
+    Map::MapType type;
+    if (directive == llvm::omp::Directive::OMPD_target_enter_data &&
+        semaCtx.langOptions().OpenMPVersion >= 52)
+      type = mapType.value_or(Map::MapType::To);
+    else if (directive == llvm::omp::Directive::OMPD_target_exit_data &&
+             semaCtx.langOptions().OpenMPVersion >= 52)
+      type = mapType.value_or(Map::MapType::From);
+    else
+      type = mapType.value_or(Map::MapType::Tofrom);
+
     switch (type) {
     case Map::MapType::To:
       mapTypeBits |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO;
diff --git a/flang/lib/Lower/OpenMP/ClauseProcessor.h b/flang/lib/Lower/OpenMP/ClauseProcessor.h
index c957a94d387e..3d8c4a337a4a 100644
--- a/flang/lib/Lower/OpenMP/ClauseProcessor.h
+++ b/flang/lib/Lower/OpenMP/ClauseProcessor.h
@@ -139,6 +139,7 @@ public:
   bool processMap(mlir::Location currentLocation,
                   lower::StatementContext &stmtCtx,
                   mlir::omp::MapClauseOps &result,
+                  llvm::omp::Directive directive = llvm::omp::OMPD_unknown,
                   llvm::SmallVectorImpl<const semantics::Symbol *> *mapSyms =
                       nullptr) const;
   bool processMotionClauses(lower::StatementContext &stmtCtx,
diff --git a/flang/lib/Lower/OpenMP/Clauses.cpp b/flang/lib/Lower/OpenMP/Clauses.cpp
index c0c57d1832d4..b599d69a3627 100644
--- a/flang/lib/Lower/OpenMP/Clauses.cpp
+++ b/flang/lib/Lower/OpenMP/Clauses.cpp
@@ -1043,7 +1043,7 @@ Map make(const parser::OmpClause::Map &inp,
   auto type = [&]() -> std::optional<Map::MapType> {
     if (t3)
       return convert1(t3->v);
-    return Map::MapType::Tofrom;
+    return std::nullopt;
   }();
 
   Map::MapTypeModifiers typeMods;
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 7ad886959727..2595a08f626e 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1831,7 +1831,8 @@ static void genTargetClauses(
   }
   cp.processIf(llvm::omp::Directive::OMPD_target, clauseOps);
   cp.processIsDevicePtr(clauseOps, isDevicePtrSyms);
-  cp.processMap(loc, stmtCtx, clauseOps, &mapSyms);
+  cp.processMap(loc, stmtCtx, clauseOps, llvm::omp::Directive::OMPD_unknown,
+                &mapSyms);
   cp.processNowait(clauseOps);
   cp.processThreadLimit(stmtCtx, clauseOps);
 
@@ -1884,7 +1885,7 @@ static void genTargetEnterExitUpdateDataClauses(
   if (directive == llvm::omp::Directive::OMPD_target_update)
     cp.processMotionClauses(stmtCtx, clauseOps);
   else
-    cp.processMap(loc, stmtCtx, clauseOps);
+    cp.processMap(loc, stmtCtx, clauseOps, directive);
 
   cp.processNowait(clauseOps);
 }
diff --git a/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90 b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
new file mode 100644
index 000000000000..0d4fd964b71e
--- /dev/null
+++ b/flang/test/Lower/OpenMP/target-enter-data-default-openmp52.f90
@@ -0,0 +1,27 @@
+! This test checks the lowering and application of default map types for the target enter/exit data constructs and map clauses
+
+!RUN: %flang -fc1 -emit-fir -fopenmp -fopenmp-version=52 -o - %s | FileCheck %s --check-prefix=CHECK-52
+!RUN: not %flang -fc1 -emit-fir -fopenmp -fopenmp-version=51 -o - %s 2>&1| FileCheck %s --check-prefix=CHECK-51
+
+module test
+  real, allocatable :: A
+
+contains
+  subroutine initialize()
+  allocate(A)
+  !$omp target enter data map(A)
+  !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, f32) map_clauses(to) capture(ByRef) var_ptr_ptr(%5 : !fir.llvm_ptr<!fir.ref<f32>>) -> !fir.llvm_ptr<!fir.ref<f32>> {name = ""}
+  !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(to) capture(ByRef) members(%6 : [0] : !fir.llvm_ptr<!fir.ref<f32>>) -> !fir.ref<!fir.box<!fir.heap<f32>>> {name = "a"}
+  !CHECK-51: to and alloc map types are permitted
+
+  end subroutine initialize
+
+  subroutine finalize()
+  !$omp target exit data map(A)
+  !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, f32) map_clauses(from) capture(ByRef) var_ptr_ptr(%3 : !fir.llvm_ptr<!fir.ref<f32>>) -> !fir.llvm_ptr<!fir.ref<f32>> {name = ""}
+  !CHECK-52: omp.map.info var_ptr(%2 : !fir.ref<!fir.box<!fir.heap<f32>>>, !fir.box<!fir.heap<f32>>) map_clauses(from) capture(ByRef) members(%4 : [0] : !fir.llvm_ptr<!fir.ref<f32>>) -> !fir.ref<!fir.box<!fir.heap<f32>>> {name = "a"}
+  !CHECK-51: from, release and delete map types are permitted
+  deallocate(A)
+  
+  end subroutine finalize
+end module test
-- 
cgit v1.2.3


From eb694b28461fdbd5e347fca59829e8a9ad021773 Mon Sep 17 00:00:00 2001
From: Tobias Gysi <tobias.gysi@nextsilicon.com>
Date: Thu, 19 Jun 2025 16:32:48 +0200
Subject: [mlir][arith] Delete mul ext canonicalizations (#144844)

The Arith dialect includes patterns that canonicalize a sequence of:

- trunci(shrui(mul(sext(x), sext(y)), c)) -> mulsi_extended(x, y)
- trunci(shrui(mul(zext(x), zext(y)), c)) -> mului_extended(x, y)

These patterns return the high word of an extended multiplication, which
assumes that the shift amount is equal to the bit width of the original
operands. This check was missing, leading to incorrect canonicalizations
when the shift amount was less than the bit width.

For example, the following code:
```
  %x = arith.extui %a: i32 to i33
  %y = arith.extui %b: i32 to i33
  %m = arith.muli %x, %y: i33
  %c1 = arith.constant 1: i33
  %sh = arith.shrui %m, %c1 : i33
  %hi = arith.trunci %sh: i33 to i32
```
would incorrectly be canonicalized to:
```
_, %hi = arith.mului_extended %a, %b : i32
```
This commit removes the faulty canonicalizations since they are not
believed to be generally beneficial (c.f., the discussion of the
alternative https://github.com/llvm/llvm-project/pull/144787 which fixes
the canonicalizations).
---
 mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td |  28 +----
 mlir/lib/Dialect/Arith/IR/ArithOps.cpp             |   6 +-
 mlir/test/Dialect/Arith/canonicalize.mlir          | 114 +--------------------
 3 files changed, 5 insertions(+), 143 deletions(-)

diff --git a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
index b61612436eb7..64eccc76a664 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
+++ b/mlir/lib/Dialect/Arith/IR/ArithCanonicalization.td
@@ -273,7 +273,7 @@ def RedundantSelectFalse :
     Pat<(SelectOp $pred, $a, (SelectOp $pred, $b, $c)),
         (SelectOp $pred, $a, $c)>;
 
-// select(pred, false, true) => not(pred) 
+// select(pred, false, true) => not(pred)
 def SelectI1ToNot :
     Pat<(SelectOp $pred,
                   (ConstantLikeMatcher ConstantAttr<I1Attr, "0">),
@@ -361,10 +361,6 @@ def OrOfExtSI :
 // TruncIOp
 //===----------------------------------------------------------------------===//
 
-def ValuesWithSameType :
-    Constraint<
-      CPred<"llvm::all_equal({$0.getType(), $1.getType(), $2.getType()})">>;
-
 def ValueWiderThan :
     Constraint<And<[
       CPred<"getScalarOrElementWidth($0) > getScalarOrElementWidth($1)">,
@@ -397,28 +393,6 @@ def TruncIShrSIToTrunciShrUI :
         (Arith_TruncIOp (Arith_ShRUIOp $x, (Arith_ConstantOp (cast<"TypedAttr"> $c0))), $overflow),
         [(TruncationMatchesShiftAmount $x, $tr, $c0)]>;
 
-// trunci(shrui(mul(sext(x), sext(y)), c)) -> mulsi_extended(x, y)
-def TruncIShrUIMulIToMulSIExtended :
-    Pat<(Arith_TruncIOp:$tr (Arith_ShRUIOp
-                              (Arith_MulIOp:$mul
-                                (Arith_ExtSIOp $x), (Arith_ExtSIOp $y), $ovf1),
-                              (ConstantLikeMatcher AnyAttr:$c0)), $overflow),
-        (Arith_MulSIExtendedOp:$res__1 $x, $y),
-      [(ValuesWithSameType $tr, $x, $y),
-       (ValueWiderThan $mul, $x),
-       (TruncationMatchesShiftAmount $mul, $x, $c0)]>;
-
-// trunci(shrui(mul(zext(x), zext(y)), c)) -> mului_extended(x, y)
-def TruncIShrUIMulIToMulUIExtended :
-    Pat<(Arith_TruncIOp:$tr (Arith_ShRUIOp
-                              (Arith_MulIOp:$mul
-                                (Arith_ExtUIOp $x), (Arith_ExtUIOp $y), $ovf1),
-                              (ConstantLikeMatcher AnyAttr:$c0)), $overflow),
-        (Arith_MulUIExtendedOp:$res__1 $x, $y),
-      [(ValuesWithSameType $tr, $x, $y),
-       (ValueWiderThan $mul, $x),
-       (TruncationMatchesShiftAmount $mul, $x, $c0)]>;
-
 //===----------------------------------------------------------------------===//
 // TruncIOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
index 9e53e195274a..12a1e18597ac 100644
--- a/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
+++ b/mlir/lib/Dialect/Arith/IR/ArithOps.cpp
@@ -1509,9 +1509,9 @@ bool arith::TruncIOp::areCastCompatible(TypeRange inputs, TypeRange outputs) {
 
 void arith::TruncIOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
                                                   MLIRContext *context) {
-  patterns.add<TruncIExtSIToExtSI, TruncIExtUIToExtUI, TruncIShrSIToTrunciShrUI,
-               TruncIShrUIMulIToMulSIExtended, TruncIShrUIMulIToMulUIExtended>(
-      context);
+  patterns
+      .add<TruncIExtSIToExtSI, TruncIExtUIToExtUI, TruncIShrSIToTrunciShrUI>(
+          context);
 }
 
 LogicalResult arith::TruncIOp::verify() {
diff --git a/mlir/test/Dialect/Arith/canonicalize.mlir b/mlir/test/Dialect/Arith/canonicalize.mlir
index b6188c81ff91..076f3a99cd39 100644
--- a/mlir/test/Dialect/Arith/canonicalize.mlir
+++ b/mlir/test/Dialect/Arith/canonicalize.mlir
@@ -1000,7 +1000,7 @@ func.func @tripleAddAddOvf2(%arg0: index) -> index {
 
 
 // CHECK-LABEL: @foldSubXX_tensor
-//       CHECK:   %[[c0:.+]] = arith.constant dense<0> : tensor<10xi32> 
+//       CHECK:   %[[c0:.+]] = arith.constant dense<0> : tensor<10xi32>
 //       CHECK:   %[[sub:.+]] = arith.subi
 //       CHECK:   return %[[c0]], %[[sub]]
 func.func @foldSubXX_tensor(%static : tensor<10xi32>, %dyn : tensor<?x?xi32>) -> (tensor<10xi32>, tensor<?x?xi32>) {
@@ -2952,118 +2952,6 @@ func.func @truncIShrSIToTrunciShrUIBadShiftAmt2(%a: i64) -> i32 {
   return %hi : i32
 }
 
-// CHECK-LABEL: @wideMulToMulSIExtended
-//  CHECK-SAME:   (%[[A:.+]]: i32, %[[B:.+]]: i32)
-//  CHECK-NEXT:   %[[LOW:.+]], %[[HIGH:.+]] = arith.mulsi_extended %[[A]], %[[B]] : i32
-//  CHECK-NEXT:   return %[[HIGH]] : i32
-func.func @wideMulToMulSIExtended(%a: i32, %b: i32) -> i32 {
-  %x = arith.extsi %a: i32 to i64
-  %y = arith.extsi %b: i32 to i64
-  %m = arith.muli %x, %y: i64
-  %c32 = arith.constant 32: i64
-  %sh = arith.shrui %m, %c32 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
-// CHECK-LABEL: @wideMulToMulSIExtendedVector
-//  CHECK-SAME:   (%[[A:.+]]: vector<3xi32>, %[[B:.+]]: vector<3xi32>)
-//  CHECK-NEXT:   %[[LOW:.+]], %[[HIGH:.+]] = arith.mulsi_extended %[[A]], %[[B]] : vector<3xi32>
-//  CHECK-NEXT:   return %[[HIGH]] : vector<3xi32>
-func.func @wideMulToMulSIExtendedVector(%a: vector<3xi32>, %b: vector<3xi32>) -> vector<3xi32> {
-  %x = arith.extsi %a: vector<3xi32> to vector<3xi64>
-  %y = arith.extsi %b: vector<3xi32> to vector<3xi64>
-  %m = arith.muli %x, %y: vector<3xi64>
-  %c32 = arith.constant dense<32>: vector<3xi64>
-  %sh = arith.shrui %m, %c32 : vector<3xi64>
-  %hi = arith.trunci %sh: vector<3xi64> to vector<3xi32>
-  return %hi : vector<3xi32>
-}
-
-// CHECK-LABEL: @wideMulToMulUIExtended
-//  CHECK-SAME:   (%[[A:.+]]: i32, %[[B:.+]]: i32)
-//  CHECK-NEXT:   %[[LOW:.+]], %[[HIGH:.+]] = arith.mului_extended %[[A]], %[[B]] : i32
-//  CHECK-NEXT:   return %[[HIGH]] : i32
-func.func @wideMulToMulUIExtended(%a: i32, %b: i32) -> i32 {
-  %x = arith.extui %a: i32 to i64
-  %y = arith.extui %b: i32 to i64
-  %m = arith.muli %x, %y: i64
-  %c32 = arith.constant 32: i64
-  %sh = arith.shrui %m, %c32 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
-// CHECK-LABEL: @wideMulToMulUIExtendedVector
-//  CHECK-SAME:   (%[[A:.+]]: vector<3xi32>, %[[B:.+]]: vector<3xi32>)
-//  CHECK-NEXT:   %[[LOW:.+]], %[[HIGH:.+]] = arith.mului_extended %[[A]], %[[B]] : vector<3xi32>
-//  CHECK-NEXT:   return %[[HIGH]] : vector<3xi32>
-func.func @wideMulToMulUIExtendedVector(%a: vector<3xi32>, %b: vector<3xi32>) -> vector<3xi32> {
-  %x = arith.extui %a: vector<3xi32> to vector<3xi64>
-  %y = arith.extui %b: vector<3xi32> to vector<3xi64>
-  %m = arith.muli %x, %y: vector<3xi64>
-  %c32 = arith.constant dense<32>: vector<3xi64>
-  %sh = arith.shrui %m, %c32 : vector<3xi64>
-  %hi = arith.trunci %sh: vector<3xi64> to vector<3xi32>
-  return %hi : vector<3xi32>
-}
-
-// CHECK-LABEL: @wideMulToMulIExtendedMixedExt
-//       CHECK:   arith.muli
-//       CHECK:   arith.shrui
-//       CHECK:   arith.trunci
-func.func @wideMulToMulIExtendedMixedExt(%a: i32, %b: i32) -> i32 {
-  %x = arith.extsi %a: i32 to i64
-  %y = arith.extui %b: i32 to i64
-  %m = arith.muli %x, %y: i64
-  %c32 = arith.constant 32: i64
-  %sh = arith.shrui %m, %c32 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
-// CHECK-LABEL: @wideMulToMulSIExtendedBadExt
-//       CHECK:   arith.muli
-//       CHECK:   arith.shrui
-//       CHECK:   arith.trunci
-func.func @wideMulToMulSIExtendedBadExt(%a: i16, %b: i16) -> i32 {
-  %x = arith.extsi %a: i16 to i64
-  %y = arith.extsi %b: i16 to i64
-  %m = arith.muli %x, %y: i64
-  %c32 = arith.constant 32: i64
-  %sh = arith.shrui %m, %c32 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
-// CHECK-LABEL: @wideMulToMulSIExtendedBadShift1
-//       CHECK:   arith.muli
-//       CHECK:   arith.shrui
-//       CHECK:   arith.trunci
-func.func @wideMulToMulSIExtendedBadShift1(%a: i32, %b: i32) -> i32 {
-  %x = arith.extsi %a: i32 to i64
-  %y = arith.extsi %b: i32 to i64
-  %m = arith.muli %x, %y: i64
-  %c33 = arith.constant 33: i64
-  %sh = arith.shrui %m, %c33 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
-// CHECK-LABEL: @wideMulToMulSIExtendedBadShift2
-//       CHECK:   arith.muli
-//       CHECK:   arith.shrui
-//       CHECK:   arith.trunci
-func.func @wideMulToMulSIExtendedBadShift2(%a: i32, %b: i32) -> i32 {
-  %x = arith.extsi %a: i32 to i64
-  %y = arith.extsi %b: i32 to i64
-  %m = arith.muli %x, %y: i64
-  %c31 = arith.constant 31: i64
-  %sh = arith.shrui %m, %c31 : i64
-  %hi = arith.trunci %sh: i64 to i32
-  return %hi : i32
-}
-
 // CHECK-LABEL: @foldShli0
 // CHECK-SAME: (%[[ARG:.*]]: i64)
 //       CHECK:   return %[[ARG]] : i64
-- 
cgit v1.2.3


From 3516ad05dfd674d731487cb67bbfe48f7e1fda9c Mon Sep 17 00:00:00 2001
From: "Mikhail R. Gadelha" <mikhail@igalia.com>
Date: Thu, 19 Jun 2025 11:42:50 -0300
Subject: [RISCV] Update SpacemiT X60 scheduling latencies based on hardware
 measurements (#144730)

This patch updates the RISC-V SpacemiT X60 scheduling model with latency
values collected from the X60 hardware. The previous values were
empirically derived but were slightly off.

  Changes:
  - LoadLatency (baseline for load instructions): 5 --> 3 cycles
  - Memory operations: unified at 4 cycles
  - Atomic loads/stores: 5 --> 8 cycles
  - Atomic RMW operations: 5 --> 12 cycles

Hardware-measured values provide more accurate instruction scheduling
for the in-order X60 core. Testing shows NFC across benchmarks except
for 523.xalancbmk_r (known to be noisy).

https://lnt.lukelau.me/db_default/v4/nts/663?compare_to=657
---
 llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td     |  15 +-
 .../test/tools/llvm-mca/RISCV/SpacemitX60/atomic.s | 176 ++++++++++-----------
 .../llvm-mca/RISCV/SpacemitX60/floating-point.s    |  12 +-
 .../tools/llvm-mca/RISCV/SpacemitX60/integer.s     |  22 +--
 4 files changed, 113 insertions(+), 112 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td b/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
index 8948694c420a..9059d5a4e497 100644
--- a/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
+++ b/llvm/lib/Target/RISCV/RISCVSchedSpacemitX60.td
@@ -16,7 +16,7 @@
 def SpacemitX60Model : SchedMachineModel {
   let IssueWidth        = 2; // dual-issue
   let MicroOpBufferSize = 0; // in-order
-  let LoadLatency       = 5; // worse case: >= 3
+  let LoadLatency       = 3; // worse case: >= 3
   let MispredictPenalty = 9; // nine-stage
 
   let CompleteModel = 0;
@@ -114,7 +114,7 @@ def : WriteRes<WriteBEXT, [SMX60_IEU]>;
 def : WriteRes<WriteBEXTI, [SMX60_IEU]>;
 
 // Memory/Atomic memory
-let Latency = 3 in {
+let Latency = 4 in {
   def : WriteRes<WriteSTB, [SMX60_LS]>;
   def : WriteRes<WriteSTH, [SMX60_LS]>;
   def : WriteRes<WriteSTW, [SMX60_LS]>;
@@ -122,11 +122,7 @@ let Latency = 3 in {
   def : WriteRes<WriteFST16, [SMX60_LS]>;
   def : WriteRes<WriteFST32, [SMX60_LS]>;
   def : WriteRes<WriteFST64, [SMX60_LS]>;
-  def : WriteRes<WriteAtomicSTW, [SMX60_LS]>;
-  def : WriteRes<WriteAtomicSTD, [SMX60_LS]>;
-}
 
-let Latency = 5 in {
   def : WriteRes<WriteLDB, [SMX60_LS]>;
   def : WriteRes<WriteLDH, [SMX60_LS]>;
   def : WriteRes<WriteLDW, [SMX60_LS]>;
@@ -137,9 +133,14 @@ let Latency = 5 in {
 }
 
 // Atomics
-let Latency = 5 in {
+let Latency = 8 in {
+  def : WriteRes<WriteAtomicSTW, [SMX60_LS]>;
+  def : WriteRes<WriteAtomicSTD, [SMX60_LS]>;
   def : WriteRes<WriteAtomicLDW, [SMX60_LS]>;
   def : WriteRes<WriteAtomicLDD, [SMX60_LS]>;
+}
+
+let Latency = 12 in {
   def : WriteRes<WriteAtomicW, [SMX60_LS]>;
   def : WriteRes<WriteAtomicD, [SMX60_LS]>;
 }
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/atomic.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/atomic.s
index ceab015e2720..bc9229471b20 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/atomic.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/atomic.s
@@ -120,94 +120,94 @@ amomaxu.d.aqrl s5, s4, (s3)
 # CHECK-NEXT: [9]: LLVM Opcode Name
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]                                        [9]                        Instructions:
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_W                       lr.w	t0, (t1)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_W_AQ                    lr.w.aq	t1, (t2)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_W_RL                    lr.w.rl	t2, (t3)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_W_AQ_RL                 lr.w.aqrl	t3, (t4)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_W                       sc.w	t6, t5, (t4)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_W_AQ                    sc.w.aq	t5, t4, (t3)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_W_RL                    sc.w.rl	t4, t3, (t2)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_W_AQ_RL                 sc.w.aqrl	t3, t2, (t1)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_D                       lr.d	t0, (t1)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_D_AQ                    lr.d.aq	t1, (t2)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_D_RL                    lr.d.rl	t2, (t3)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LR_D_AQ_RL                 lr.d.aqrl	t3, (t4)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_D                       sc.d	t6, t5, (t4)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_D_AQ                    sc.d.aq	t5, t4, (t3)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_D_RL                    sc.d.rl	t4, t3, (t2)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SC_D_AQ_RL                 sc.d.aqrl	t3, t2, (t1)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_W                  amoswap.w	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_W                   amoadd.w	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_W                   amoxor.w	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_W                   amoand.w	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_W                    amoor.w	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_W                   amomin.w	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_W                   amomax.w	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_W                  amominu.w	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_W                  amomaxu.w	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_W_AQ               amoswap.w.aq	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_W_AQ                amoadd.w.aq	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_W_AQ                amoxor.w.aq	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_W_AQ                amoand.w.aq	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_W_AQ                 amoor.w.aq	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_W_AQ                amomin.w.aq	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_W_AQ                amomax.w.aq	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_W_AQ               amominu.w.aq	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_W_AQ               amomaxu.w.aq	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_W_RL               amoswap.w.rl	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_W_RL                amoadd.w.rl	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_W_RL                amoxor.w.rl	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_W_RL                amoand.w.rl	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_W_RL                 amoor.w.rl	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_W_RL                amomin.w.rl	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_W_RL                amomax.w.rl	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_W_RL               amominu.w.rl	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_W_RL               amomaxu.w.rl	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_W_AQ_RL            amoswap.w.aqrl	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_W_AQ_RL             amoadd.w.aqrl	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_W_AQ_RL             amoxor.w.aqrl	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_W_AQ_RL             amoand.w.aqrl	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_W_AQ_RL              amoor.w.aqrl	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_W_AQ_RL             amomin.w.aqrl	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_W_AQ_RL             amomax.w.aqrl	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_W_AQ_RL            amominu.w.aqrl	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_W_AQ_RL            amomaxu.w.aqrl	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_D                  amoswap.d	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_D                   amoadd.d	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_D                   amoxor.d	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_D                   amoand.d	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_D                    amoor.d	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_D                   amomin.d	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_D                   amomax.d	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_D                  amominu.d	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_D                  amomaxu.d	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_D_AQ               amoswap.d.aq	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_D_AQ                amoadd.d.aq	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_D_AQ                amoxor.d.aq	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_D_AQ                amoand.d.aq	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_D_AQ                 amoor.d.aq	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_D_AQ                amomin.d.aq	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_D_AQ                amomax.d.aq	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_D_AQ               amominu.d.aq	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_D_AQ               amomaxu.d.aq	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_D_RL               amoswap.d.rl	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_D_RL                amoadd.d.rl	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_D_RL                amoxor.d.rl	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_D_RL                amoand.d.rl	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_D_RL                 amoor.d.rl	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_D_RL                amomin.d.rl	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_D_RL                amomax.d.rl	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_D_RL               amominu.d.rl	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_D_RL               amomaxu.d.rl	s5, s4, (s3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOSWAP_D_AQ_RL            amoswap.d.aqrl	a4, ra, (s0)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOADD_D_AQ_RL             amoadd.d.aqrl	a1, a2, (a3)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOXOR_D_AQ_RL             amoxor.d.aqrl	a2, a3, (a4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOAND_D_AQ_RL             amoand.d.aqrl	a3, a4, (a5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOOR_D_AQ_RL              amoor.d.aqrl	a4, a5, (a6)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMIN_D_AQ_RL             amomin.d.aqrl	a5, a6, (a7)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAX_D_AQ_RL             amomax.d.aqrl	s7, s6, (s5)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMINU_D_AQ_RL            amominu.d.aqrl	s6, s5, (s4)
-# CHECK-NEXT:  1      5     0.50    *      *             5     SMX60_LS                                   AMOMAXU_D_AQ_RL            amomaxu.d.aqrl	s5, s4, (s3)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_W                       lr.w	t0, (t1)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_W_AQ                    lr.w.aq	t1, (t2)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_W_RL                    lr.w.rl	t2, (t3)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_W_AQ_RL                 lr.w.aqrl	t3, (t4)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_W                       sc.w	t6, t5, (t4)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_W_AQ                    sc.w.aq	t5, t4, (t3)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_W_RL                    sc.w.rl	t4, t3, (t2)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_W_AQ_RL                 sc.w.aqrl	t3, t2, (t1)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_D                       lr.d	t0, (t1)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_D_AQ                    lr.d.aq	t1, (t2)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_D_RL                    lr.d.rl	t2, (t3)
+# CHECK-NEXT:  1      8     0.50    *                    8     SMX60_LS                                   LR_D_AQ_RL                 lr.d.aqrl	t3, (t4)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_D                       sc.d	t6, t5, (t4)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_D_AQ                    sc.d.aq	t5, t4, (t3)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_D_RL                    sc.d.rl	t4, t3, (t2)
+# CHECK-NEXT:  1      8     0.50           *             8     SMX60_LS                                   SC_D_AQ_RL                 sc.d.aqrl	t3, t2, (t1)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_W                  amoswap.w	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_W                   amoadd.w	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_W                   amoxor.w	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_W                   amoand.w	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_W                    amoor.w	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_W                   amomin.w	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_W                   amomax.w	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_W                  amominu.w	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_W                  amomaxu.w	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_W_AQ               amoswap.w.aq	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_W_AQ                amoadd.w.aq	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_W_AQ                amoxor.w.aq	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_W_AQ                amoand.w.aq	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_W_AQ                 amoor.w.aq	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_W_AQ                amomin.w.aq	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_W_AQ                amomax.w.aq	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_W_AQ               amominu.w.aq	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_W_AQ               amomaxu.w.aq	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_W_RL               amoswap.w.rl	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_W_RL                amoadd.w.rl	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_W_RL                amoxor.w.rl	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_W_RL                amoand.w.rl	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_W_RL                 amoor.w.rl	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_W_RL                amomin.w.rl	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_W_RL                amomax.w.rl	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_W_RL               amominu.w.rl	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_W_RL               amomaxu.w.rl	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_W_AQ_RL            amoswap.w.aqrl	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_W_AQ_RL             amoadd.w.aqrl	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_W_AQ_RL             amoxor.w.aqrl	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_W_AQ_RL             amoand.w.aqrl	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_W_AQ_RL              amoor.w.aqrl	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_W_AQ_RL             amomin.w.aqrl	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_W_AQ_RL             amomax.w.aqrl	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_W_AQ_RL            amominu.w.aqrl	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_W_AQ_RL            amomaxu.w.aqrl	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_D                  amoswap.d	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_D                   amoadd.d	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_D                   amoxor.d	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_D                   amoand.d	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_D                    amoor.d	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_D                   amomin.d	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_D                   amomax.d	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_D                  amominu.d	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_D                  amomaxu.d	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_D_AQ               amoswap.d.aq	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_D_AQ                amoadd.d.aq	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_D_AQ                amoxor.d.aq	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_D_AQ                amoand.d.aq	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_D_AQ                 amoor.d.aq	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_D_AQ                amomin.d.aq	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_D_AQ                amomax.d.aq	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_D_AQ               amominu.d.aq	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_D_AQ               amomaxu.d.aq	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_D_RL               amoswap.d.rl	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_D_RL                amoadd.d.rl	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_D_RL                amoxor.d.rl	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_D_RL                amoand.d.rl	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_D_RL                 amoor.d.rl	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_D_RL                amomin.d.rl	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_D_RL                amomax.d.rl	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_D_RL               amominu.d.rl	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_D_RL               amomaxu.d.rl	s5, s4, (s3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOSWAP_D_AQ_RL            amoswap.d.aqrl	a4, ra, (s0)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOADD_D_AQ_RL             amoadd.d.aqrl	a1, a2, (a3)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOXOR_D_AQ_RL             amoxor.d.aqrl	a2, a3, (a4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOAND_D_AQ_RL             amoand.d.aqrl	a3, a4, (a5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOOR_D_AQ_RL              amoor.d.aqrl	a4, a5, (a6)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMIN_D_AQ_RL             amomin.d.aqrl	a5, a6, (a7)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAX_D_AQ_RL             amomax.d.aqrl	s7, s6, (s5)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMINU_D_AQ_RL            amominu.d.aqrl	s6, s5, (s4)
+# CHECK-NEXT:  1      12    0.50    *      *             12    SMX60_LS                                   AMOMAXU_D_AQ_RL            amomaxu.d.aqrl	s5, s4, (s3)
 
 # CHECK:      Resources:
 # CHECK-NEXT: [0]   - SMX60_FP
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/floating-point.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/floating-point.s
index bd3666ef7bb9..b86fcbccbeab 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/floating-point.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/floating-point.s
@@ -148,12 +148,12 @@ fclass.d a3, ft10
 # CHECK-NEXT: [9]: LLVM Opcode Name
 
 # CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    [7]    [8]                                        [9]                        Instructions:
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   FLH                        flh	ft0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   FSH                        fsh	ft0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   FLW                        flw	ft0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   FSW                        fsw	ft0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   FLD                        fld	ft0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   FSD                        fsd	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   FLH                        flh	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   FSH                        fsh	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   FLW                        flw	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   FSW                        fsw	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   FLD                        fld	ft0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   FSD                        fsd	ft0, 0(a0)
 # CHECK-NEXT:  1      4     1.00                         4     SMX60_FP                                   FADD_H                     fadd.h	fs10, fs11, ft8
 # CHECK-NEXT:  1      4     1.00                         4     SMX60_FP                                   FSUB_H                     fsub.h	ft9, ft10, ft11
 # CHECK-NEXT:  1      4     1.00                         4     SMX60_FP                                   FMUL_H                     fmul.h	ft0, ft1, ft2
diff --git a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/integer.s b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/integer.s
index 8b43874499f2..b72540f29f48 100644
--- a/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/integer.s
+++ b/llvm/test/tools/llvm-mca/RISCV/SpacemitX60/integer.s
@@ -222,17 +222,17 @@ bseti a0, a1, 1
 # CHECK-NEXT:  1      1     1.00                         1     SMX60_IEU,SMX60_IEUA                       BGE                        bge	a0, a0, .Ltmp5
 # CHECK-NEXT:  1      1     1.00                         1     SMX60_IEU,SMX60_IEUA                       BGEU                       bgeu	a0, a0, .Ltmp6
 # CHECK-NEXT:  1      1     0.50                         1     SMX60_IEU                                  C_ADD                      add	a0, a0, a0
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LB                         lb	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LBU                        lbu	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LH                         lh	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LHU                        lhu	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LW                         lw	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LWU                        lwu	t0, 0(a0)
-# CHECK-NEXT:  1      5     0.50    *                    5     SMX60_LS                                   LD                         ld	t0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SB                         sb	t0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SH                         sh	t0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SW                         sw	t0, 0(a0)
-# CHECK-NEXT:  1      3     0.50           *             3     SMX60_LS                                   SD                         sd	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LB                         lb	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LBU                        lbu	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LH                         lh	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LHU                        lhu	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LW                         lw	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LWU                        lwu	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50    *                    4     SMX60_LS                                   LD                         ld	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   SB                         sb	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   SH                         sh	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   SW                         sw	t0, 0(a0)
+# CHECK-NEXT:  1      4     0.50           *             4     SMX60_LS                                   SD                         sd	t0, 0(a0)
 # CHECK-NEXT:  1      6     0.50                         6     SMX60_IEU                                  MUL                        mul	a0, a0, a0
 # CHECK-NEXT:  1      6     0.50                         6     SMX60_IEU                                  MULH                       mulh	a0, a0, a0
 # CHECK-NEXT:  1      6     0.50                         6     SMX60_IEU                                  MULHU                      mulhu	a0, a0, a0
-- 
cgit v1.2.3


From fdb572681de7e12b7cd03fd33478022272eb1bae Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Thu, 19 Jun 2025 15:50:53 +0100
Subject: Fix bazel build after #144594 (#144904)

---
 utils/bazel/llvm-project-overlay/llvm/BUILD.bazel | 41 +++++++++++++++++++++--
 1 file changed, 38 insertions(+), 3 deletions(-)

diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 31855cd5444c..d5a7fdb747f8 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -418,6 +418,7 @@ cc_library(
         "include/llvm/BinaryFormat/ELFRelocs/*.def",
     ]),
     deps = [
+        ":PPCTargetParser",
         ":Support",
         ":TargetParser",
     ],
@@ -1364,6 +1365,13 @@ td_library(
     ]),
 )
 
+td_library(
+    name = "PPCTargetTdFiles",
+    srcs = glob([
+        "lib/Target/PowerPC/**/*.td",
+    ]),
+)
+
 gentbl_cc_library(
     name = "RISCVTargetParserDefGen",
     tbl_outs = {"include/llvm/TargetParser/RISCVTargetParserDef.inc": ["-gen-riscv-target-def"]},
@@ -1375,11 +1383,38 @@ gentbl_cc_library(
     ],
 )
 
+gentbl_cc_library(
+    name = "PPCGenTargetFeaturesGen",
+    tbl_outs = {"include/llvm/TargetParser/PPCGenTargetFeatures.inc": ["-gen-target-features"]},
+    tblgen = ":llvm-tblgen",
+    td_file = "lib/Target/PowerPC/PPC.td",
+    deps = [
+        ":CommonTargetTdFiles",
+        ":PPCTargetTdFiles",
+    ],
+)
+
+cc_library(
+    name = "PPCTargetParser",
+    srcs = ["lib/TargetParser/PPCTargetParser.cpp"],
+    hdrs = ["include/llvm/TargetParser/PPCTargetParser.h"],
+    copts = llvm_copts,
+    includes = ["include"],
+    deps = [
+        ":PPCGenTargetFeaturesGen",
+        ":Support",
+        ":TargetParser",
+    ],
+)
+
 cc_library(
     name = "TargetParser",
-    srcs = glob([
-        "lib/TargetParser/*.cpp",
-    ]) + select({
+    srcs = glob(
+        [
+            "lib/TargetParser/*.cpp",
+        ],
+        exclude = ["lib/TargetParser/PPCTargetParser.cpp"],
+    ) + select({
         "@platforms//os:windows": glob([
             "lib/TargetParser/Windows/*.inc",
         ]),
-- 
cgit v1.2.3


From 5eb24fde11cd82a08f208509f80f428da90c89c9 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Thu, 19 Jun 2025 08:06:07 -0700
Subject: [SelectionDAG][RISCV] Preserve nneg flag when folding (trunc (zext
 X))->(zext X). (#144807)

If X is known non-negative, that's still true if we fold the truncate
to create a smaller zext.

In the i128 tests, SelectionDAGBuilder aggressively truncates the
`zext nneg` to i64 to match `getShiftAmountTy`. If we don't preserve
the `nneg` we can't see that the shift amount argument being `signext`
means we don't need to do any extension
---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp  |   8 +-
 llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp |   8 +-
 llvm/test/CodeGen/RISCV/shifts.ll              | 295 +++++++++++++++++++++++++
 3 files changed, 307 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0e078f9dd88b..a6b9cc81edde 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -15740,8 +15740,12 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
       N0.getOpcode() == ISD::SIGN_EXTEND ||
       N0.getOpcode() == ISD::ANY_EXTEND) {
     // if the source is smaller than the dest, we still need an extend.
-    if (N0.getOperand(0).getValueType().bitsLT(VT))
-      return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0));
+    if (N0.getOperand(0).getValueType().bitsLT(VT)) {
+      SDNodeFlags Flags;
+      if (N0.getOpcode() == ISD::ZERO_EXTEND)
+        Flags.setNonNeg(N0->getFlags().hasNonNeg());
+      return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), Flags);
+    }
     // if the source is larger than the dest, than we just need the truncate.
     if (N0.getOperand(0).getValueType().bitsGT(VT))
       return DAG.getNode(ISD::TRUNCATE, DL, VT, N0.getOperand(0));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index b0e3f534e2aa..5d8db8be9731 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -6474,8 +6474,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
         OpOpcode == ISD::ANY_EXTEND) {
       // If the source is smaller than the dest, we still need an extend.
       if (N1.getOperand(0).getValueType().getScalarType().bitsLT(
-              VT.getScalarType()))
-        return getNode(OpOpcode, DL, VT, N1.getOperand(0));
+              VT.getScalarType())) {
+        SDNodeFlags Flags;
+        if (OpOpcode == ISD::ZERO_EXTEND)
+          Flags.setNonNeg(N1->getFlags().hasNonNeg());
+        return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
+      }
       if (N1.getOperand(0).getValueType().bitsGT(VT))
         return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
       return N1.getOperand(0);
diff --git a/llvm/test/CodeGen/RISCV/shifts.ll b/llvm/test/CodeGen/RISCV/shifts.ll
index 249dabba0cc2..32a037918a5a 100644
--- a/llvm/test/CodeGen/RISCV/shifts.ll
+++ b/llvm/test/CodeGen/RISCV/shifts.ll
@@ -484,3 +484,298 @@ define i128 @fshr128_minsize(i128 %a, i128 %b) minsize nounwind {
   %res = tail call i128 @llvm.fshr.i128(i128 %a, i128 %a, i128 %b)
   ret i128 %res
 }
+
+define i64 @lshr64_shamt32(i64 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: lshr64_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a4, a2, -32
+; RV32I-NEXT:    srl a3, a1, a2
+; RV32I-NEXT:    bltz a4, .LBB11_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a0, a3
+; RV32I-NEXT:    j .LBB11_3
+; RV32I-NEXT:  .LBB11_2:
+; RV32I-NEXT:    srl a0, a0, a2
+; RV32I-NEXT:    not a2, a2
+; RV32I-NEXT:    slli a1, a1, 1
+; RV32I-NEXT:    sll a1, a1, a2
+; RV32I-NEXT:    or a0, a0, a1
+; RV32I-NEXT:  .LBB11_3:
+; RV32I-NEXT:    srai a1, a4, 31
+; RV32I-NEXT:    and a1, a1, a3
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: lshr64_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    srl a0, a0, a1
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i64
+  %1 = lshr i64 %a, %zext
+  ret i64 %1
+}
+
+define i64 @ashr64_shamt32(i64 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: ashr64_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    mv a3, a1
+; RV32I-NEXT:    addi a4, a2, -32
+; RV32I-NEXT:    sra a1, a1, a2
+; RV32I-NEXT:    bltz a4, .LBB12_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    srai a3, a3, 31
+; RV32I-NEXT:    mv a0, a1
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB12_2:
+; RV32I-NEXT:    srl a0, a0, a2
+; RV32I-NEXT:    not a2, a2
+; RV32I-NEXT:    slli a3, a3, 1
+; RV32I-NEXT:    sll a2, a3, a2
+; RV32I-NEXT:    or a0, a0, a2
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ashr64_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sra a0, a0, a1
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i64
+  %1 = ashr i64 %a, %zext
+  ret i64 %1
+}
+
+define i64 @shl64_shamt32(i64 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: shl64_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi a4, a2, -32
+; RV32I-NEXT:    sll a3, a0, a2
+; RV32I-NEXT:    bltz a4, .LBB13_2
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    mv a1, a3
+; RV32I-NEXT:    j .LBB13_3
+; RV32I-NEXT:  .LBB13_2:
+; RV32I-NEXT:    sll a1, a1, a2
+; RV32I-NEXT:    not a2, a2
+; RV32I-NEXT:    srli a0, a0, 1
+; RV32I-NEXT:    srl a0, a0, a2
+; RV32I-NEXT:    or a1, a1, a0
+; RV32I-NEXT:  .LBB13_3:
+; RV32I-NEXT:    srai a0, a4, 31
+; RV32I-NEXT:    and a0, a0, a3
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: shl64_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    sll a0, a0, a1
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i64
+  %1 = shl i64 %a, %zext
+  ret i64 %1
+}
+
+define i128 @lshr128_shamt32(i128 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: lshr128_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    sw zero, 16(sp)
+; RV32I-NEXT:    sw zero, 20(sp)
+; RV32I-NEXT:    sw zero, 24(sp)
+; RV32I-NEXT:    sw zero, 28(sp)
+; RV32I-NEXT:    srli a6, a2, 3
+; RV32I-NEXT:    mv a7, sp
+; RV32I-NEXT:    andi t0, a2, 31
+; RV32I-NEXT:    andi a6, a6, 12
+; RV32I-NEXT:    xori t0, t0, 31
+; RV32I-NEXT:    add a6, a7, a6
+; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    lw a1, 0(a6)
+; RV32I-NEXT:    lw a3, 4(a6)
+; RV32I-NEXT:    lw a4, 8(a6)
+; RV32I-NEXT:    lw a5, 12(a6)
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    slli a6, a3, 1
+; RV32I-NEXT:    srl a3, a3, a2
+; RV32I-NEXT:    slli a7, a4, 1
+; RV32I-NEXT:    srl a4, a4, a2
+; RV32I-NEXT:    srl a2, a5, a2
+; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    sll a6, a6, t0
+; RV32I-NEXT:    sll a7, a7, t0
+; RV32I-NEXT:    sll a5, a5, t0
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a3, a3, a7
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: lshr128_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a4, a2, -64
+; RV64I-NEXT:    srl a3, a1, a2
+; RV64I-NEXT:    bltz a4, .LBB14_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a0, a3
+; RV64I-NEXT:    j .LBB14_3
+; RV64I-NEXT:  .LBB14_2:
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    not a2, a2
+; RV64I-NEXT:    slli a1, a1, 1
+; RV64I-NEXT:    sll a1, a1, a2
+; RV64I-NEXT:    or a0, a0, a1
+; RV64I-NEXT:  .LBB14_3:
+; RV64I-NEXT:    srai a1, a4, 63
+; RV64I-NEXT:    and a1, a1, a3
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i128
+  %1 = lshr i128 %a, %zext
+  ret i128 %1
+}
+
+define i128 @ashr128_shamt32(i128 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: ashr128_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    srli a6, a2, 3
+; RV32I-NEXT:    mv a7, sp
+; RV32I-NEXT:    andi t0, a2, 31
+; RV32I-NEXT:    andi a6, a6, 12
+; RV32I-NEXT:    xori t0, t0, 31
+; RV32I-NEXT:    add a6, a7, a6
+; RV32I-NEXT:    sw a3, 0(sp)
+; RV32I-NEXT:    sw a4, 4(sp)
+; RV32I-NEXT:    sw a5, 8(sp)
+; RV32I-NEXT:    sw a1, 12(sp)
+; RV32I-NEXT:    srai a1, a1, 31
+; RV32I-NEXT:    sw a1, 16(sp)
+; RV32I-NEXT:    sw a1, 20(sp)
+; RV32I-NEXT:    sw a1, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    lw a1, 0(a6)
+; RV32I-NEXT:    lw a3, 4(a6)
+; RV32I-NEXT:    lw a4, 8(a6)
+; RV32I-NEXT:    lw a5, 12(a6)
+; RV32I-NEXT:    srl a1, a1, a2
+; RV32I-NEXT:    slli a6, a3, 1
+; RV32I-NEXT:    srl a3, a3, a2
+; RV32I-NEXT:    slli a7, a4, 1
+; RV32I-NEXT:    srl a4, a4, a2
+; RV32I-NEXT:    sra a2, a5, a2
+; RV32I-NEXT:    slli a5, a5, 1
+; RV32I-NEXT:    sll a6, a6, t0
+; RV32I-NEXT:    sll a7, a7, t0
+; RV32I-NEXT:    sll a5, a5, t0
+; RV32I-NEXT:    or a1, a1, a6
+; RV32I-NEXT:    or a3, a3, a7
+; RV32I-NEXT:    or a4, a4, a5
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a3, 4(a0)
+; RV32I-NEXT:    sw a4, 8(a0)
+; RV32I-NEXT:    sw a2, 12(a0)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: ashr128_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    mv a3, a1
+; RV64I-NEXT:    addi a4, a2, -64
+; RV64I-NEXT:    sra a1, a1, a2
+; RV64I-NEXT:    bltz a4, .LBB15_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    srai a3, a3, 63
+; RV64I-NEXT:    mv a0, a1
+; RV64I-NEXT:    mv a1, a3
+; RV64I-NEXT:    ret
+; RV64I-NEXT:  .LBB15_2:
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    not a2, a2
+; RV64I-NEXT:    slli a3, a3, 1
+; RV64I-NEXT:    sll a2, a3, a2
+; RV64I-NEXT:    or a0, a0, a2
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i128
+  %1 = ashr i128 %a, %zext
+  ret i128 %1
+}
+
+define i128 @shl128_shamt32(i128 %a, i32 signext %b) nounwind {
+; RV32I-LABEL: shl128_shamt32:
+; RV32I:       # %bb.0:
+; RV32I-NEXT:    addi sp, sp, -32
+; RV32I-NEXT:    lw a3, 0(a1)
+; RV32I-NEXT:    lw a4, 4(a1)
+; RV32I-NEXT:    lw a5, 8(a1)
+; RV32I-NEXT:    lw a1, 12(a1)
+; RV32I-NEXT:    sw zero, 0(sp)
+; RV32I-NEXT:    sw zero, 4(sp)
+; RV32I-NEXT:    sw zero, 8(sp)
+; RV32I-NEXT:    sw zero, 12(sp)
+; RV32I-NEXT:    srli a6, a2, 3
+; RV32I-NEXT:    addi a7, sp, 16
+; RV32I-NEXT:    andi t0, a2, 31
+; RV32I-NEXT:    andi a6, a6, 12
+; RV32I-NEXT:    sub a6, a7, a6
+; RV32I-NEXT:    sw a3, 16(sp)
+; RV32I-NEXT:    sw a4, 20(sp)
+; RV32I-NEXT:    sw a5, 24(sp)
+; RV32I-NEXT:    sw a1, 28(sp)
+; RV32I-NEXT:    lw a1, 0(a6)
+; RV32I-NEXT:    lw a3, 4(a6)
+; RV32I-NEXT:    lw a4, 8(a6)
+; RV32I-NEXT:    lw a5, 12(a6)
+; RV32I-NEXT:    xori a6, t0, 31
+; RV32I-NEXT:    sll a7, a3, a2
+; RV32I-NEXT:    srli t0, a1, 1
+; RV32I-NEXT:    sll a5, a5, a2
+; RV32I-NEXT:    sll a1, a1, a2
+; RV32I-NEXT:    sll a2, a4, a2
+; RV32I-NEXT:    srli a3, a3, 1
+; RV32I-NEXT:    srli a4, a4, 1
+; RV32I-NEXT:    srl t0, t0, a6
+; RV32I-NEXT:    srl a3, a3, a6
+; RV32I-NEXT:    srl a4, a4, a6
+; RV32I-NEXT:    or a6, a7, t0
+; RV32I-NEXT:    or a2, a2, a3
+; RV32I-NEXT:    or a4, a5, a4
+; RV32I-NEXT:    sw a1, 0(a0)
+; RV32I-NEXT:    sw a6, 4(a0)
+; RV32I-NEXT:    sw a2, 8(a0)
+; RV32I-NEXT:    sw a4, 12(a0)
+; RV32I-NEXT:    addi sp, sp, 32
+; RV32I-NEXT:    ret
+;
+; RV64I-LABEL: shl128_shamt32:
+; RV64I:       # %bb.0:
+; RV64I-NEXT:    addi a4, a2, -64
+; RV64I-NEXT:    sll a3, a0, a2
+; RV64I-NEXT:    bltz a4, .LBB16_2
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    mv a1, a3
+; RV64I-NEXT:    j .LBB16_3
+; RV64I-NEXT:  .LBB16_2:
+; RV64I-NEXT:    sll a1, a1, a2
+; RV64I-NEXT:    not a2, a2
+; RV64I-NEXT:    srli a0, a0, 1
+; RV64I-NEXT:    srl a0, a0, a2
+; RV64I-NEXT:    or a1, a1, a0
+; RV64I-NEXT:  .LBB16_3:
+; RV64I-NEXT:    srai a0, a4, 63
+; RV64I-NEXT:    and a0, a0, a3
+; RV64I-NEXT:    ret
+  %zext = zext nneg i32 %b to i128
+  %1 = shl i128 %a, %zext
+  ret i128 %1
+}
-- 
cgit v1.2.3


From 3de01d07c33c10dfefc753c87c0a926fd512425b Mon Sep 17 00:00:00 2001
From: Karlo Basioli <k.basioli@gmail.com>
Date: Thu, 19 Jun 2025 16:16:03 +0100
Subject: Fix bazel build after #144594, mark variable as potentially unused
 (#144910)

---
 llvm/lib/TargetParser/TargetParser.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 03f7d3899c2e..90791dfb7b7c 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -56,7 +56,7 @@ std::optional<llvm::StringMap<bool>> llvm::getCPUDefaultTargetFeatures(
   llvm::StringMap<bool> DefaultFeatures;
   setImpliedBits(Bits, CPUEntry->Implies.getAsBitset(), ProcFeatures);
 
-  unsigned BitSize = Bits.size();
+  [[maybe_unused]] unsigned BitSize = Bits.size();
   for (const BasicSubtargetFeatureKV &FE : ProcFeatures) {
     assert(FE.Value < BitSize && "Target Feature is out of range");
     if (Bits[FE.Value])
-- 
cgit v1.2.3


From 36af7345dfb8e84a1f2971db34089b63321e8467 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer@sony.com>
Date: Thu, 15 May 2025 11:43:49 +0100
Subject: Reapply "[Clang] Enable -fextend-variable-liveness at -Og (#118026)"

Relands this feature after several fixes:

* Force fake uses to be emitted before musttail calls (#136867)
* Added soften-float legalization for fake uses (#142714)
* Treat fake uses as size-less instructions in a SystemZ assert (#144390)

If further issues with fake uses are found then this may be reverted again,
but all currently-known issues are resolved.

This reverts commit 2dc6e98169baeb1f73036da0ea50fd828d8323d0.
---
 clang/docs/CommandGuide/clang.rst            | 7 +++++--
 clang/docs/ReleaseNotes.rst                  | 4 ++++
 clang/lib/Driver/ToolChains/Clang.cpp        | 8 +++++++-
 clang/test/Driver/extend-variable-liveness.c | 3 ++-
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/clang/docs/CommandGuide/clang.rst b/clang/docs/CommandGuide/clang.rst
index 1b8776c5e9ad..7d49f2cc28a1 100644
--- a/clang/docs/CommandGuide/clang.rst
+++ b/clang/docs/CommandGuide/clang.rst
@@ -460,8 +460,11 @@ Code Generation Options
     :option:`-Oz` Like :option:`-Os` (and thus :option:`-O2`), but reduces code
     size further.
 
-    :option:`-Og` Like :option:`-O1`. In future versions, this option might
-    disable different optimizations in order to improve debuggability.
+    :option:`-Og` Similar to :option:`-O1`, but with slightly reduced
+    optimization and better variable visibility. The same optimizations are run
+    as at :option:`-O1`, but the ``-fextend-variable-liveness`` flag is
+    also set, which tries to prevent optimizations from reducing the liveness of
+    user variables, improving their availability when debugging.
 
     :option:`-O` Equivalent to :option:`-O1`.
 
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index dd748ab06873..96477ef6ddc9 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -366,6 +366,10 @@ Modified Compiler Flags
 
 - The ``-fveclib=libmvec`` option now supports AArch64 targets (requires GLIBC 2.40 or newer).
 
+- The ``-Og`` optimization flag now sets ``-fextend-variable-liveness``,
+  reducing performance slightly while reducing the number of optimized-out
+  variables. (#GH118026)
+
 Removed Compiler Flags
 -------------------------
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a78a1c897818..5a3c09e3a343 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7555,7 +7555,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Args.hasArg(options::OPT_fretain_comments_from_system_headers))
     CmdArgs.push_back("-fretain-comments-from-system-headers");
 
-  Args.AddLastArg(CmdArgs, options::OPT_fextend_variable_liveness_EQ);
+  if (Arg *A = Args.getLastArg(options::OPT_fextend_variable_liveness_EQ)) {
+    A->render(Args, CmdArgs);
+  } else if (Arg *A = Args.getLastArg(options::OPT_O_Group);
+             A && A->containsValue("g")) {
+    // Set -fextend-variable-liveness=all by default at -Og.
+    CmdArgs.push_back("-fextend-variable-liveness=all");
+  }
 
   // Forward -fcomment-block-commands to -cc1.
   Args.AddAllArgs(CmdArgs, options::OPT_fcomment_block_commands);
diff --git a/clang/test/Driver/extend-variable-liveness.c b/clang/test/Driver/extend-variable-liveness.c
index bbfb2ece6f29..99a5409cecce 100644
--- a/clang/test/Driver/extend-variable-liveness.c
+++ b/clang/test/Driver/extend-variable-liveness.c
@@ -1,7 +1,8 @@
 // Tests that -fextend-variable-liveness and its aliases are correctly passed
-// by the driver.
+// by the driver, and are set by default at -Og.
 
 // RUN: %clang -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,DEFAULT
+// RUN: %clang -### -Og -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL
 // RUN: %clang -fextend-variable-liveness=none -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,NONE
 // RUN: %clang -fextend-variable-liveness=this -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,THIS
 // RUN: %clang -fextend-variable-liveness=all -### -c %s 2>&1 | FileCheck %s --check-prefixes=CHECK,ALL
-- 
cgit v1.2.3


From f4db14229cd975822c41376afda9d56a29f9396c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Thu, 19 Jun 2025 17:26:54 +0200
Subject: [SCCP] Move logic for removing ssa.copy into Solver (NFC)

So it can be reused between IPSCCP and SCCP.

Make the implementation a bit more efficient by only lookup the
PredicateInfo once.
---
 llvm/include/llvm/Transforms/Utils/SCCPSolver.h |  2 ++
 llvm/lib/Transforms/IPO/SCCP.cpp                | 14 +-------------
 llvm/lib/Transforms/Utils/SCCPSolver.cpp        | 24 ++++++++++++++++++++++++
 3 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
index c8fe4301b2b4..5aac7c2ac5d3 100644
--- a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
+++ b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
@@ -77,6 +77,8 @@ public:
   LLVM_ABI void addPredicateInfo(Function &F, DominatorTree &DT,
                                  AssumptionCache &AC);
 
+  LLVM_ABI void removeSSACopies(Function &F);
+
   /// markBlockExecutable - This method can be used by clients to mark all of
   /// the blocks that are known to be intrinsically live in the processed unit.
   /// This returns true if the block was not considered live before.
diff --git a/llvm/lib/Transforms/IPO/SCCP.cpp b/llvm/lib/Transforms/IPO/SCCP.cpp
index 43c5df357500..d50de34dfa48 100644
--- a/llvm/lib/Transforms/IPO/SCCP.cpp
+++ b/llvm/lib/Transforms/IPO/SCCP.cpp
@@ -250,19 +250,7 @@ static bool runIPSCCP(
       if (!DeadBB->hasAddressTaken())
         DTU.deleteBB(DeadBB);
 
-    for (BasicBlock &BB : F) {
-      for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-        if (Solver.getPredicateInfoFor(&Inst)) {
-          if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-            if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
-              Value *Op = II->getOperand(0);
-              Inst.replaceAllUsesWith(Op);
-              Inst.eraseFromParent();
-            }
-          }
-        }
-      }
-    }
+    Solver.removeSSACopies(F);
   }
 
   // If we inferred constant or undef return values for a function, we replaced
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index f4b378b82dae..cc0bb4735c23 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -764,6 +764,26 @@ public:
     FnPredicateInfo.insert({&F, std::make_unique<PredicateInfo>(F, DT, AC)});
   }
 
+  void removeSSACopies(Function &F) {
+    auto It = FnPredicateInfo.find(&F);
+    if (It == FnPredicateInfo.end())
+      return;
+
+    for (BasicBlock &BB : F) {
+      for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
+        if (It->second->getPredicateInfoFor(&Inst)) {
+          if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
+            if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+              Value *Op = II->getOperand(0);
+              Inst.replaceAllUsesWith(Op);
+              Inst.eraseFromParent();
+            }
+          }
+        }
+      }
+    }
+  }
+
   void visitCallInst(CallInst &I) { visitCallBase(I); }
 
   bool markBlockExecutable(BasicBlock *BB);
@@ -2168,6 +2188,10 @@ void SCCPSolver::addPredicateInfo(Function &F, DominatorTree &DT,
   Visitor->addPredicateInfo(F, DT, AC);
 }
 
+void SCCPSolver::removeSSACopies(Function &F) {
+  Visitor->removeSSACopies(F);
+}
+
 bool SCCPSolver::markBlockExecutable(BasicBlock *BB) {
   return Visitor->markBlockExecutable(BB);
 }
-- 
cgit v1.2.3


From 01d648a42939c834b6b45677e540882222b01c11 Mon Sep 17 00:00:00 2001
From: Steven Perron <stevenperron@google.com>
Date: Thu, 19 Jun 2025 11:52:55 -0400
Subject: [HLSL][SPIRV] Reapply "[HLSL][SPIRV] Add vk::constant_id attribute."
 (#144902)

- **Reapply "[HLSL][SPIRV] Add vk::constant_id attribute." (#144812)**
- **Fix memory leak.**
---
 clang/include/clang/Basic/Attr.td                  |   8 +
 clang/include/clang/Basic/AttrDocs.td              |  15 ++
 clang/include/clang/Basic/Builtins.td              |  13 ++
 clang/include/clang/Basic/DiagnosticSemaKinds.td   |   4 +
 clang/include/clang/Sema/SemaHLSL.h                |   5 +-
 clang/lib/CodeGen/CGHLSLBuiltins.cpp               |  75 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h                |   6 +
 clang/lib/Sema/SemaDecl.cpp                        |  13 ++
 clang/lib/Sema/SemaDeclAttr.cpp                    |   3 +
 clang/lib/Sema/SemaHLSL.cpp                        | 120 +++++++++++-
 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl    | 130 +++++++++++++
 .../inline-spirv/SpirvType.alignment.hlsl          |  16 --
 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl |  68 -------
 .../vk-features/SpirvType.alignment.hlsl           |  16 ++
 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl  |  68 +++++++
 .../CodeGenHLSL/vk-features/vk.spec-constant.hlsl  | 210 +++++++++++++++++++++
 clang/test/SemaHLSL/vk.spec-constant.error.hlsl    |  37 ++++
 17 files changed, 721 insertions(+), 86 deletions(-)
 create mode 100644 clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
 delete mode 100644 clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
 create mode 100644 clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
 create mode 100644 clang/test/SemaHLSL/vk.spec-constant.error.hlsl

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index f113cd2ba2fb..27fea7dea0a5 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -5023,6 +5023,14 @@ def HLSLVkExtBuiltinInput : InheritableAttr {
   let Documentation = [HLSLVkExtBuiltinInputDocs];
 }
 
+def HLSLVkConstantId : InheritableAttr {
+  let Spellings = [CXX11<"vk", "constant_id">];
+  let Args = [IntArgument<"Id">];
+  let Subjects = SubjectList<[ExternalGlobalVar]>;
+  let LangOpts = [HLSL];
+  let Documentation = [VkConstantIdDocs];
+}
+
 def RandomizeLayout : InheritableAttr {
   let Spellings = [GCC<"randomize_layout">];
   let Subjects = SubjectList<[Record]>;
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index 6051e1fc4511..43442f177ab7 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -8252,6 +8252,21 @@ and https://microsoft.github.io/hlsl-specs/proposals/0013-wave-size-range.html
   }];
 }
 
+def VkConstantIdDocs : Documentation {
+  let Category = DocCatFunction;
+  let Content = [{
+The ``vk::constant_id`` attribute specifies the id for a SPIR-V specialization
+constant. The attribute applies to const global scalar variables. The variable must be initialized with a C++11 constexpr.
+In SPIR-V, the
+variable will be replaced with an `OpSpecConstant` with the given id.
+The syntax is:
+
+.. code-block:: text
+
+  ``[[vk::constant_id(<Id>)]] const T Name = <Init>``
+}];
+}
+
 def RootSignatureDocs : Documentation {
   let Category = DocCatFunction;
   let Content = [{
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 68cd3d790e78..d65b3a5d2f44 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5065,6 +5065,19 @@ def HLSLGroupMemoryBarrierWithGroupSync: LangBuiltin<"HLSL_LANG"> {
   let Prototype = "void()";
 }
 
+class HLSLScalarTemplate
+    : Template<["bool", "char", "short", "int", "long long int",
+                "unsigned short", "unsigned int", "unsigned long long int",
+                "__fp16", "float", "double"],
+               ["_bool", "_char", "_short", "_int", "_longlong", "_ushort",
+                "_uint", "_ulonglong", "_half", "_float", "_double"]>;
+
+def HLSLGetSpirvSpecConstant : LangBuiltin<"HLSL_LANG">, HLSLScalarTemplate {
+  let Spellings = ["__builtin_get_spirv_spec_constant"];
+  let Attributes = [NoThrow, Const, Pure];
+  let Prototype = "T(unsigned int, T)";
+}
+
 // Builtins for XRay.
 def XRayCustomEvent : Builtin {
   let Spellings = ["__xray_customevent"];
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 979ff60b73b7..34b798a09c21 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12927,6 +12927,10 @@ def err_spirv_enum_not_int : Error<
 def err_spirv_enum_not_valid : Error<
    "invalid value for %select{storage class}0 argument">;
 
+def err_specialization_const
+    : Error<"variable with 'vk::constant_id' attribute must be a const "
+            "int/float/enum/bool and be initialized with a literal">;
+
 // errors of expect.with.probability
 def err_probability_not_constant_float : Error<
    "probability argument to __builtin_expect_with_probability must be constant "
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 33c4b8d1568b..97091792ba23 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -98,6 +98,8 @@ public:
   HLSLWaveSizeAttr *mergeWaveSizeAttr(Decl *D, const AttributeCommonInfo &AL,
                                       int Min, int Max, int Preferred,
                                       int SpelledArgsCount);
+  HLSLVkConstantIdAttr *
+  mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL, int Id);
   HLSLShaderAttr *mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                                   llvm::Triple::EnvironmentType ShaderType);
   HLSLParamModifierAttr *
@@ -135,6 +137,7 @@ public:
   void handleRootSignatureAttr(Decl *D, const ParsedAttr &AL);
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
   void handleWaveSizeAttr(Decl *D, const ParsedAttr &AL);
+  void handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_DispatchThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupThreadIDAttr(Decl *D, const ParsedAttr &AL);
   void handleSV_GroupIDAttr(Decl *D, const ParsedAttr &AL);
@@ -171,7 +174,7 @@ public:
   QualType getInoutParameterType(QualType Ty);
 
   bool transformInitList(const InitializedEntity &Entity, InitListExpr *Init);
-
+  bool handleInitialization(VarDecl *VDecl, Expr *&Init);
   void deduceAddressSpace(VarDecl *Decl);
 
 private:
diff --git a/clang/lib/CodeGen/CGHLSLBuiltins.cpp b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
index ccf45c0c6ff1..2a60a0909c93 100644
--- a/clang/lib/CodeGen/CGHLSLBuiltins.cpp
+++ b/clang/lib/CodeGen/CGHLSLBuiltins.cpp
@@ -12,6 +12,7 @@
 
 #include "CGBuiltin.h"
 #include "CGHLSLRuntime.h"
+#include "CodeGenFunction.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -214,6 +215,44 @@ static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,
   }
 }
 
+// Returns the mangled name for a builtin function that the SPIR-V backend
+// will expand into a spec Constant.
+static std::string getSpecConstantFunctionName(clang::QualType SpecConstantType,
+                                               ASTContext &Context) {
+  // The parameter types for our conceptual intrinsic function.
+  QualType ClangParamTypes[] = {Context.IntTy, SpecConstantType};
+
+  // Create a temporary FunctionDecl for the builtin fuction. It won't be
+  // added to the AST.
+  FunctionProtoType::ExtProtoInfo EPI;
+  QualType FnType =
+      Context.getFunctionType(SpecConstantType, ClangParamTypes, EPI);
+  DeclarationName FuncName = &Context.Idents.get("__spirv_SpecConstant");
+  FunctionDecl *FnDeclForMangling = FunctionDecl::Create(
+      Context, Context.getTranslationUnitDecl(), SourceLocation(),
+      SourceLocation(), FuncName, FnType, /*TSI=*/nullptr, SC_Extern);
+
+  // Attach the created parameter declarations to the function declaration.
+  SmallVector<ParmVarDecl *, 2> ParamDecls;
+  for (QualType ParamType : ClangParamTypes) {
+    ParmVarDecl *PD = ParmVarDecl::Create(
+        Context, FnDeclForMangling, SourceLocation(), SourceLocation(),
+        /*IdentifierInfo*/ nullptr, ParamType, /*TSI*/ nullptr, SC_None,
+        /*DefaultArg*/ nullptr);
+    ParamDecls.push_back(PD);
+  }
+  FnDeclForMangling->setParams(ParamDecls);
+
+  // Get the mangled name.
+  std::string Name;
+  llvm::raw_string_ostream MangledNameStream(Name);
+  std::unique_ptr<MangleContext> Mangler(Context.createMangleContext());
+  Mangler->mangleName(FnDeclForMangling, MangledNameStream);
+  MangledNameStream.flush();
+
+  return Name;
+}
+
 Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
                                             const CallExpr *E,
                                             ReturnValueSlot ReturnValue) {
@@ -773,6 +812,42 @@ Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
     return EmitRuntimeCall(
         Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
   }
+  case Builtin::BI__builtin_get_spirv_spec_constant_bool:
+  case Builtin::BI__builtin_get_spirv_spec_constant_short:
+  case Builtin::BI__builtin_get_spirv_spec_constant_ushort:
+  case Builtin::BI__builtin_get_spirv_spec_constant_int:
+  case Builtin::BI__builtin_get_spirv_spec_constant_uint:
+  case Builtin::BI__builtin_get_spirv_spec_constant_longlong:
+  case Builtin::BI__builtin_get_spirv_spec_constant_ulonglong:
+  case Builtin::BI__builtin_get_spirv_spec_constant_half:
+  case Builtin::BI__builtin_get_spirv_spec_constant_float:
+  case Builtin::BI__builtin_get_spirv_spec_constant_double: {
+    llvm::Function *SpecConstantFn = getSpecConstantFunction(E->getType());
+    llvm::Value *SpecId = EmitScalarExpr(E->getArg(0));
+    llvm::Value *DefaultVal = EmitScalarExpr(E->getArg(1));
+    llvm::Value *Args[] = {SpecId, DefaultVal};
+    return Builder.CreateCall(SpecConstantFn, Args);
+  }
   }
   return nullptr;
 }
+
+llvm::Function *clang::CodeGen::CodeGenFunction::getSpecConstantFunction(
+    const clang::QualType &SpecConstantType) {
+
+  // Find or create the declaration for the function.
+  llvm::Module *M = &CGM.getModule();
+  std::string MangledName =
+      getSpecConstantFunctionName(SpecConstantType, getContext());
+  llvm::Function *SpecConstantFn = M->getFunction(MangledName);
+
+  if (!SpecConstantFn) {
+    llvm::Type *IntType = ConvertType(getContext().IntTy);
+    llvm::Type *RetTy = ConvertType(SpecConstantType);
+    llvm::Type *ArgTypes[] = {IntType, RetTy};
+    llvm::FunctionType *FnTy = llvm::FunctionType::get(RetTy, ArgTypes, false);
+    SpecConstantFn = llvm::Function::Create(
+        FnTy, llvm::GlobalValue::ExternalLinkage, MangledName, M);
+  }
+  return SpecConstantFn;
+}
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index a5ab9df01dba..59f14b3e35fd 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -4850,6 +4850,12 @@ public:
   llvm::Value *EmitAMDGPUBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitHLSLBuiltinExpr(unsigned BuiltinID, const CallExpr *E,
                                    ReturnValueSlot ReturnValue);
+
+  // Returns a builtin function that the SPIR-V backend will expand into a spec
+  // constant.
+  llvm::Function *
+  getSpecConstantFunction(const clang::QualType &SpecConstantType);
+
   llvm::Value *EmitDirectXBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitSPIRVBuiltinExpr(unsigned BuiltinID, const CallExpr *E);
   llvm::Value *EmitScalarOrConstFoldImmArg(unsigned ICEArguments, unsigned Idx,
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 1bf72e5bb7b9..e1cccf068b5a 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -2890,6 +2890,8 @@ static bool mergeDeclAttribute(Sema &S, NamedDecl *D,
     NewAttr = S.HLSL().mergeWaveSizeAttr(D, *WS, WS->getMin(), WS->getMax(),
                                          WS->getPreferred(),
                                          WS->getSpelledArgsCount());
+  else if (const auto *CI = dyn_cast<HLSLVkConstantIdAttr>(Attr))
+    NewAttr = S.HLSL().mergeVkConstantIdAttr(D, *CI, CI->getId());
   else if (const auto *SA = dyn_cast<HLSLShaderAttr>(Attr))
     NewAttr = S.HLSL().mergeShaderAttr(D, *SA, SA->getType());
   else if (isa<SuppressAttr>(Attr))
@@ -13757,6 +13759,10 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init, bool DirectInit) {
     return;
   }
 
+  if (getLangOpts().HLSL)
+    if (!HLSL().handleInitialization(VDecl, Init))
+      return;
+
   // Get the decls type and save a reference for later, since
   // CheckInitializerTypes may change it.
   QualType DclT = VDecl->getType(), SavT = DclT;
@@ -14179,6 +14185,13 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl) {
       }
     }
 
+    // HLSL variable with the `vk::constant_id` attribute must be initialized.
+    if (!Var->isInvalidDecl() && Var->hasAttr<HLSLVkConstantIdAttr>()) {
+      Diag(Var->getLocation(), diag::err_specialization_const);
+      Var->setInvalidDecl();
+      return;
+    }
+
     if (!Var->isInvalidDecl() && RealDecl->hasAttr<LoaderUninitializedAttr>()) {
       if (Var->getStorageClass() == SC_Extern) {
         Diag(Var->getLocation(), diag::err_loader_uninitialized_extern_decl)
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 1c2fa80e782d..eba29e609cb0 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -7590,6 +7590,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL,
   case ParsedAttr::AT_HLSLVkExtBuiltinInput:
     S.HLSL().handleVkExtBuiltinInputAttr(D, AL);
     break;
+  case ParsedAttr::AT_HLSLVkConstantId:
+    S.HLSL().handleVkConstantIdAttr(D, AL);
+    break;
   case ParsedAttr::AT_HLSLSV_GroupThreadID:
     S.HLSL().handleSV_GroupThreadIDAttr(D, AL);
     break;
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index b55f4fd786b5..9b43ee00810b 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -119,6 +119,40 @@ static ResourceClass getResourceClass(RegisterType RT) {
   llvm_unreachable("unexpected RegisterType value");
 }
 
+static Builtin::ID getSpecConstBuiltinId(QualType Type) {
+  const auto *BT = dyn_cast<BuiltinType>(Type);
+  if (!BT) {
+    if (!Type->isEnumeralType())
+      return Builtin::NotBuiltin;
+    return Builtin::BI__builtin_get_spirv_spec_constant_int;
+  }
+
+  switch (BT->getKind()) {
+  case BuiltinType::Bool:
+    return Builtin::BI__builtin_get_spirv_spec_constant_bool;
+  case BuiltinType::Short:
+    return Builtin::BI__builtin_get_spirv_spec_constant_short;
+  case BuiltinType::Int:
+    return Builtin::BI__builtin_get_spirv_spec_constant_int;
+  case BuiltinType::LongLong:
+    return Builtin::BI__builtin_get_spirv_spec_constant_longlong;
+  case BuiltinType::UShort:
+    return Builtin::BI__builtin_get_spirv_spec_constant_ushort;
+  case BuiltinType::UInt:
+    return Builtin::BI__builtin_get_spirv_spec_constant_uint;
+  case BuiltinType::ULongLong:
+    return Builtin::BI__builtin_get_spirv_spec_constant_ulonglong;
+  case BuiltinType::Half:
+    return Builtin::BI__builtin_get_spirv_spec_constant_half;
+  case BuiltinType::Float:
+    return Builtin::BI__builtin_get_spirv_spec_constant_float;
+  case BuiltinType::Double:
+    return Builtin::BI__builtin_get_spirv_spec_constant_double;
+  default:
+    return Builtin::NotBuiltin;
+  }
+}
+
 DeclBindingInfo *ResourceBindings::addDeclBindingInfo(const VarDecl *VD,
                                                       ResourceClass ResClass) {
   assert(getDeclBindingInfo(VD, ResClass) == nullptr &&
@@ -607,6 +641,41 @@ HLSLWaveSizeAttr *SemaHLSL::mergeWaveSizeAttr(Decl *D,
   return Result;
 }
 
+HLSLVkConstantIdAttr *
+SemaHLSL::mergeVkConstantIdAttr(Decl *D, const AttributeCommonInfo &AL,
+                                int Id) {
+
+  auto &TargetInfo = getASTContext().getTargetInfo();
+  if (TargetInfo.getTriple().getArch() != llvm::Triple::spirv) {
+    Diag(AL.getLoc(), diag::warn_attribute_ignored) << AL;
+    return nullptr;
+  }
+
+  auto *VD = cast<VarDecl>(D);
+
+  if (getSpecConstBuiltinId(VD->getType()) == Builtin::NotBuiltin) {
+    Diag(VD->getLocation(), diag::err_specialization_const);
+    return nullptr;
+  }
+
+  if (!VD->getType().isConstQualified()) {
+    Diag(VD->getLocation(), diag::err_specialization_const);
+    return nullptr;
+  }
+
+  if (HLSLVkConstantIdAttr *CI = D->getAttr<HLSLVkConstantIdAttr>()) {
+    if (CI->getId() != Id) {
+      Diag(CI->getLocation(), diag::err_hlsl_attribute_param_mismatch) << AL;
+      Diag(AL.getLoc(), diag::note_conflicting_attribute);
+    }
+    return nullptr;
+  }
+
+  HLSLVkConstantIdAttr *Result =
+      ::new (getASTContext()) HLSLVkConstantIdAttr(getASTContext(), AL, Id);
+  return Result;
+}
+
 HLSLShaderAttr *
 SemaHLSL::mergeShaderAttr(Decl *D, const AttributeCommonInfo &AL,
                           llvm::Triple::EnvironmentType ShaderType) {
@@ -1157,6 +1226,15 @@ void SemaHLSL::handleVkExtBuiltinInputAttr(Decl *D, const ParsedAttr &AL) {
                  HLSLVkExtBuiltinInputAttr(getASTContext(), AL, ID));
 }
 
+void SemaHLSL::handleVkConstantIdAttr(Decl *D, const ParsedAttr &AL) {
+  uint32_t Id;
+  if (!SemaRef.checkUInt32Argument(AL, AL.getArgAsExpr(0), Id))
+    return;
+  HLSLVkConstantIdAttr *NewAttr = mergeVkConstantIdAttr(D, AL, Id);
+  if (NewAttr)
+    D->addAttr(NewAttr);
+}
+
 bool SemaHLSL::diagnoseInputIDType(QualType T, const ParsedAttr &AL) {
   const auto *VT = T->getAs<VectorType>();
 
@@ -3206,6 +3284,7 @@ static bool IsDefaultBufferConstantDecl(VarDecl *VD) {
   return VD->getDeclContext()->isTranslationUnit() &&
          QT.getAddressSpace() == LangAS::Default &&
          VD->getStorageClass() != SC_Static &&
+         !VD->hasAttr<HLSLVkConstantIdAttr>() &&
          !isInvalidConstantBufferLeafElementType(QT.getTypePtr());
 }
 
@@ -3273,7 +3352,8 @@ void SemaHLSL::ActOnVariableDeclarator(VarDecl *VD) {
     const Type *VarType = VD->getType().getTypePtr();
     while (VarType->isArrayType())
       VarType = VarType->getArrayElementTypeNoTypeQual();
-    if (VarType->isHLSLResourceRecord()) {
+    if (VarType->isHLSLResourceRecord() ||
+        VD->hasAttr<HLSLVkConstantIdAttr>()) {
       // Make the variable for resources static. The global externally visible
       // storage is accessed through the handle, which is a member. The variable
       // itself is not externally visible.
@@ -3696,3 +3776,41 @@ bool SemaHLSL::transformInitList(const InitializedEntity &Entity,
     Init->updateInit(Ctx, I, NewInit->getInit(I));
   return true;
 }
+
+bool SemaHLSL::handleInitialization(VarDecl *VDecl, Expr *&Init) {
+  const HLSLVkConstantIdAttr *ConstIdAttr =
+      VDecl->getAttr<HLSLVkConstantIdAttr>();
+  if (!ConstIdAttr)
+    return true;
+
+  ASTContext &Context = SemaRef.getASTContext();
+
+  APValue InitValue;
+  if (!Init->isCXX11ConstantExpr(Context, &InitValue)) {
+    Diag(VDecl->getLocation(), diag::err_specialization_const);
+    VDecl->setInvalidDecl();
+    return false;
+  }
+
+  Builtin::ID BID = getSpecConstBuiltinId(VDecl->getType());
+
+  // Argument 1: The ID from the attribute
+  int ConstantID = ConstIdAttr->getId();
+  llvm::APInt IDVal(Context.getIntWidth(Context.IntTy), ConstantID);
+  Expr *IdExpr = IntegerLiteral::Create(Context, IDVal, Context.IntTy,
+                                        ConstIdAttr->getLocation());
+
+  SmallVector<Expr *, 2> Args = {IdExpr, Init};
+  Expr *C = SemaRef.BuildBuiltinCallExpr(Init->getExprLoc(), BID, Args);
+  if (C->getType()->getCanonicalTypeUnqualified() !=
+      VDecl->getType()->getCanonicalTypeUnqualified()) {
+    C = SemaRef
+            .BuildCStyleCastExpr(SourceLocation(),
+                                 Context.getTrivialTypeSourceInfo(
+                                     Init->getType(), Init->getExprLoc()),
+                                 SourceLocation(), C)
+            .get();
+  }
+  Init = C;
+  return true;
+}
diff --git a/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
new file mode 100644
index 000000000000..c0955c1ea7b4
--- /dev/null
+++ b/clang/test/AST/HLSL/vk.spec-constant.usage.hlsl
@@ -0,0 +1,130 @@
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-unknown-vulkan-compute -x hlsl -ast-dump -o - %s | FileCheck %s
+
+// CHECK: VarDecl {{.*}} bool_const 'const hlsl_private bool' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'bool'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'bool (*)(unsigned int, bool) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'bool (unsigned int, bool) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_bool' 'bool (unsigned int, bool) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 1
+// CHECK-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' true
+[[vk::constant_id(1)]]
+const bool bool_const = true;
+
+// CHECK: VarDecl {{.*}} short_const 'const hlsl_private short' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'short'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short (*)(unsigned int, short) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'short (unsigned int, short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_short' 'short (unsigned int, short) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 2
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'short' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
+[[vk::constant_id(2)]]
+const short short_const = 4;
+
+// CHECK: VarDecl {{.*}} int_const 'const hlsl_private int' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 3
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
+[[vk::constant_id(3)]]
+const int int_const = 5;
+
+// CHECK: VarDecl {{.*}} long_const 'const hlsl_private long long' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'long long'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long (*)(unsigned int, long long) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'long long (unsigned int, long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_longlong' 'long long (unsigned int, long long) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 4
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'long long' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+[[vk::constant_id(4)]]
+const long long long_const = 8;
+
+// CHECK: VarDecl {{.*}} ushort_const 'const hlsl_private unsigned short' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned short'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short (*)(unsigned int, unsigned short) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned short (unsigned int, unsigned short) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ushort' 'unsigned short (unsigned int, unsigned short) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 5
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned short' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10
+[[vk::constant_id(5)]]
+const unsigned short ushort_const = 10;
+
+// CHECK: VarDecl {{.*}} uint_const 'const hlsl_private unsigned int' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int (*)(unsigned int, unsigned int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned int (unsigned int, unsigned int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_uint' 'unsigned int (unsigned int, unsigned int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 6
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 12
+[[vk::constant_id(6)]]
+const unsigned int uint_const = 12;
+
+
+// CHECK: VarDecl {{.*}} ulong_const 'const hlsl_private unsigned long long' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'unsigned long long'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long (*)(unsigned int, unsigned long long) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'unsigned long long (unsigned int, unsigned long long) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_ulonglong' 'unsigned long long (unsigned int, unsigned long long) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 7
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned long long' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 25
+[[vk::constant_id(7)]]
+const unsigned long long ulong_const = 25;
+
+// CHECK: VarDecl {{.*}} half_const 'const hlsl_private half' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'half'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half (*)(unsigned int, half) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'half (unsigned int, half) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_half' 'half (unsigned int, half) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'half' <FloatingCast>
+// CHECK-NEXT: FloatingLiteral {{.*}} 'float' 4.040000e+01
+[[vk::constant_id(8)]]
+const half half_const = 40.4;
+
+// CHECK: VarDecl {{.*}} float_const 'const hlsl_private float' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'float'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float (*)(unsigned int, float) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'float (unsigned int, float) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_float' 'float (unsigned int, float) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 8
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'float' <IntegralToFloating>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 50
+[[vk::constant_id(8)]]
+const float float_const = 50;
+
+// CHECK: VarDecl {{.*}} double_const 'const hlsl_private double' static cinit
+// CHECK-NEXT: CallExpr {{.*}} 'double'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double (*)(unsigned int, double) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'double (unsigned int, double) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_double' 'double (unsigned int, double) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 9
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'double' <IntegralToFloating>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 100
+[[vk::constant_id(9)]]
+const double double_const = 100;
+
+// CHECK: VarDecl {{.*}} enum_const 'const hlsl_private E' static cinit
+// CHECK-NEXT: CStyleCastExpr {{.*}} 'E' <IntegralCast>
+// CHECK-NEXT: CallExpr {{.*}} 'int'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int (*)(unsigned int, int) noexcept' <FunctionToPointerDecay>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'int (unsigned int, int) noexcept' lvalue Function {{.*}} '__builtin_get_spirv_spec_constant_int' 'int (unsigned int, int) noexcept'
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'unsigned int' <IntegralCast>
+// CHECK-NEXT: IntegerLiteral {{.*}} 'int' 10 
+// CHECK-NEXT: ImplicitCastExpr {{.*}} 'int' <IntegralCast>
+// CHECK-NEXT: DeclRefExpr {{.*}} 'E' EnumConstant {{.*}} 'e2' 'E' 
+enum E {
+    e0 = 10,
+    e1 = 20,
+    e2 = 30
+};
+
+[[vk::constant_id(10)]]
+const E enum_const = e2;
+
+// CHECK-NOT: CXXRecordDecl {{.*}} implicit struct __cblayout_$Globals definition
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
deleted file mode 100644
index 41cdd7d21bcb..000000000000
--- a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.alignment.hlsl
+++ /dev/null
@@ -1,16 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
-
-// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
-struct S {
-    int a;
-    Int b;
-};
-
-[numthreads(1,1,1)]
-void main() {
-    S value;
-}
diff --git a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl b/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
deleted file mode 100644
index 7149be0122f4..000000000000
--- a/clang/test/CodeGenHLSL/inline-spirv/SpirvType.hlsl
+++ /dev/null
@@ -1,68 +0,0 @@
-// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
-// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
-// RUN:   -o - | FileCheck %s
-
-template<class T, uint64_t Size>
-using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
-
-template<uint64_t Size>
-using ArrayBuffer = Array<RWBuffer<float>, Size>;
-
-typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
-
-typedef Array<Int, 5> ArrayInt;
-
-// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
-struct S {
-    ArrayBuffer<4> b;
-    Int i;
-};
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
-ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
-    return v;
-}
-
-// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
-Int getInt(Int v) {
-    return v;
-}
-
-// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
-// ArrayBuffer<4> g_buffers;
-// Int g_word;
-
-[numthreads(1, 1, 1)]
-void main() {
-    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    ArrayBuffer<4> buffers;
-
-    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
-    ArrayBuffer<0x123456789> longBuffers;
-
-    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-    Int word;
-
-    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
-    Int words[4];
-
-    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
-    ArrayInt words2;
-
-    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
-    S value;
-
-    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
-    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
-
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
-    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
-    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
-
-    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
-    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
-    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
-    Int word2 = getInt(word);
-}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
new file mode 100644
index 000000000000..41cdd7d21bcb
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/SpirvType.alignment.hlsl
@@ -0,0 +1,16 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+using Int = vk::SpirvType</* OpTypeInt */ 21, 4, 64, vk::Literal<vk::integral_constant<uint, 8>>, vk::Literal<vk::integral_constant<bool, false>>>;
+
+// CHECK: %struct.S = type <{ i32, target("spirv.Type", target("spirv.Literal", 8), target("spirv.Literal", 0), 21, 4, 64), [4 x i8] }>
+struct S {
+    int a;
+    Int b;
+};
+
+[numthreads(1,1,1)]
+void main() {
+    S value;
+}
diff --git a/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl b/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
new file mode 100644
index 000000000000..7149be0122f4
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/SpirvType.hlsl
@@ -0,0 +1,68 @@
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+template<class T, uint64_t Size>
+using Array = vk::SpirvOpaqueType</* OpTypeArray */ 28, T, vk::integral_constant<uint64_t, Size>>;
+
+template<uint64_t Size>
+using ArrayBuffer = Array<RWBuffer<float>, Size>;
+
+typedef vk::SpirvType</* OpTypeInt */ 21, 4, 32, vk::Literal<vk::integral_constant<uint, 32>>, vk::Literal<vk::integral_constant<bool, false>>> Int;
+
+typedef Array<Int, 5> ArrayInt;
+
+// CHECK: %struct.S = type { target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) }
+struct S {
+    ArrayBuffer<4> b;
+    Int i;
+};
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %v) #0
+ArrayBuffer<4> getArrayBuffer(ArrayBuffer<4> v) {
+    return v;
+}
+
+// CHECK: define hidden spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %v) #0
+Int getInt(Int v) {
+    return v;
+}
+
+// TODO: uncomment and test once CBuffer handles are implemented for SPIR-V
+// ArrayBuffer<4> g_buffers;
+// Int g_word;
+
+[numthreads(1, 1, 1)]
+void main() {
+    // CHECK: [[buffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    ArrayBuffer<4> buffers;
+
+    // CHECK: [[longBuffers:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 591751049, 1), 28, 0, 0), align 4
+    ArrayBuffer<0x123456789> longBuffers;
+
+    // CHECK: [[word:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+    Int word;
+
+    // CHECK: [[words:%.*]] = alloca [4 x target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32)], align 4
+    Int words[4];
+
+    // CHECK: [[words2:%.*]] = alloca target("spirv.Type", target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), target("spirv.IntegralConstant", i64, 5), 28, 0, 0), align 4
+    ArrayInt words2;
+
+    // CHECK: [[value:%.*]] = alloca %struct.S, align 1
+    S value;
+
+    // CHECK: [[buffers2:%.*]] = alloca target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), align 4
+    // CHECK: [[word2:%.*]] = alloca target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), align 4
+
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0), ptr [[buffers]], align 4
+    // CHECK: %call1 = call spir_func target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) @_Z14getArrayBufferu17spirv_type_28_0_0U5_TypeN4hlsl8RWBufferIfEEU6_ConstLm4E(target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Image", float, 5, 2, 0, 0, 2, 0), target("spirv.IntegralConstant", i64, 4), 28, 0, 0) %call1, ptr [[buffers2]], align 4
+    ArrayBuffer<4> buffers2 = getArrayBuffer(buffers);
+
+    // CHECK: [[loaded:%[0-9]+]] = load target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32), ptr [[word]], align 4
+    // CHECK: %call2 = call spir_func target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) @_Z6getIntu18spirv_type_21_4_32U4_LitLi32EU4_LitLi0E(target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) [[loaded]])
+    // CHECK: store target("spirv.Type", target("spirv.Literal", 32), target("spirv.Literal", 0), 21, 4, 32) %call2, ptr [[word2]], align 4
+    Int word2 = getInt(word);
+}
diff --git a/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
new file mode 100644
index 000000000000..cbc1fa61eae2
--- /dev/null
+++ b/clang/test/CodeGenHLSL/vk-features/vk.spec-constant.hlsl
@@ -0,0 +1,210 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-globals all --include-generated-funcs --version 5
+// RUN: %clang_cc1 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan-compute %s -emit-llvm -disable-llvm-passes \
+// RUN:   -o - | FileCheck %s
+
+[[vk::constant_id(1)]]
+const bool bool_const = true;
+
+[[vk::constant_id(1)]]
+const short short_const = 4;
+
+[[vk::constant_id(3)]]
+const int int_const = 5;
+
+[[vk::constant_id(4)]]
+const long long long_const = 8;
+
+[[vk::constant_id(5)]]
+const unsigned short ushort_const = 10;
+
+[[vk::constant_id(6)]]
+const unsigned int uint_const = 12;
+
+[[vk::constant_id(7)]]
+const unsigned long long ulong_const = 25;
+
+[[vk::constant_id(8)]]
+const half half_const = 40.4;
+
+[[vk::constant_id(8)]]
+const float float_const = 50.5;
+
+[[vk::constant_id(9)]]
+const double double_const = 100.2;
+
+enum E {
+    e0 = 10,
+    e1 = 20,
+    e2 = 30
+};
+
+[[vk::constant_id(10)]]
+const E enum_const = e2;
+
+[numthreads(1,1,1)]
+void main() {
+    bool b = bool_const;
+    short s = short_const;
+    int i = int_const;
+    long long l = long_const;
+    unsigned short us = ushort_const;
+    unsigned int ui = uint_const;
+    unsigned long long ul = ulong_const;
+    half h = half_const;
+    float f = float_const;
+    double d = double_const;
+    E e = enum_const;
+}
+//.
+// CHECK: @_ZL10bool_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL11short_const = internal addrspace(10) global i16 0, align 2
+// CHECK: @_ZL9int_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL10long_const = internal addrspace(10) global i64 0, align 8
+// CHECK: @_ZL12ushort_const = internal addrspace(10) global i16 0, align 2
+// CHECK: @_ZL10uint_const = internal addrspace(10) global i32 0, align 4
+// CHECK: @_ZL11ulong_const = internal addrspace(10) global i64 0, align 8
+// CHECK: @_ZL10half_const = internal addrspace(10) global float 0.000000e+00, align 4
+// CHECK: @_ZL11float_const = internal addrspace(10) global float 0.000000e+00, align 4
+// CHECK: @_ZL12double_const = internal addrspace(10) global double 0.000000e+00, align 8
+// CHECK: @_ZL10enum_const = internal addrspace(10) global i32 0, align 4
+//.
+// CHECK-LABEL: define internal spir_func void @_Z4mainv(
+// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[B:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[S:%.*]] = alloca i16, align 2
+// CHECK-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[L:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[US:%.*]] = alloca i16, align 2
+// CHECK-NEXT:    [[UI:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[UL:%.*]] = alloca i64, align 8
+// CHECK-NEXT:    [[H:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[F:%.*]] = alloca float, align 4
+// CHECK-NEXT:    [[D:%.*]] = alloca double, align 8
+// CHECK-NEXT:    [[E:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr addrspace(10) @_ZL10bool_const, align 4
+// CHECK-NEXT:    [[LOADEDV:%.*]] = trunc i32 [[TMP1]] to i1
+// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[LOADEDV]] to i32
+// CHECK-NEXT:    store i32 [[STOREDV]], ptr [[B]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i16, ptr addrspace(10) @_ZL11short_const, align 2
+// CHECK-NEXT:    store i16 [[TMP2]], ptr [[S]], align 2
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr addrspace(10) @_ZL9int_const, align 4
+// CHECK-NEXT:    store i32 [[TMP3]], ptr [[I]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr addrspace(10) @_ZL10long_const, align 8
+// CHECK-NEXT:    store i64 [[TMP4]], ptr [[L]], align 8
+// CHECK-NEXT:    [[TMP5:%.*]] = load i16, ptr addrspace(10) @_ZL12ushort_const, align 2
+// CHECK-NEXT:    store i16 [[TMP5]], ptr [[US]], align 2
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr addrspace(10) @_ZL10uint_const, align 4
+// CHECK-NEXT:    store i32 [[TMP6]], ptr [[UI]], align 4
+// CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr addrspace(10) @_ZL11ulong_const, align 8
+// CHECK-NEXT:    store i64 [[TMP7]], ptr [[UL]], align 8
+// CHECK-NEXT:    [[TMP8:%.*]] = load float, ptr addrspace(10) @_ZL10half_const, align 4
+// CHECK-NEXT:    store float [[TMP8]], ptr [[H]], align 4
+// CHECK-NEXT:    [[TMP9:%.*]] = load float, ptr addrspace(10) @_ZL11float_const, align 4
+// CHECK-NEXT:    store float [[TMP9]], ptr [[F]], align 4
+// CHECK-NEXT:    [[TMP10:%.*]] = load double, ptr addrspace(10) @_ZL12double_const, align 8
+// CHECK-NEXT:    store double [[TMP10]], ptr [[D]], align 8
+// CHECK-NEXT:    [[TMP11:%.*]] = load i32, ptr addrspace(10) @_ZL10enum_const, align 4
+// CHECK-NEXT:    store i32 [[TMP11]], ptr [[E]], align 4
+// CHECK-NEXT:    ret void
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init(
+// CHECK-SAME: ) #[[ATTR3:[0-9]+]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i1 @_Z20__spirv_SpecConstantib(i32 1, i1 true)
+// CHECK-NEXT:    [[STOREDV:%.*]] = zext i1 [[TMP1]] to i32
+// CHECK-NEXT:    store i32 [[STOREDV]], ptr addrspace(10) @_ZL10bool_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.1(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantis(i32 1, i16 4)
+// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL11short_const, align 2
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.2(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 3, i32 5)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL9int_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.3(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantix(i32 4, i64 8)
+// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL10long_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.4(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i16 @_Z20__spirv_SpecConstantit(i32 5, i16 10)
+// CHECK-NEXT:    store i16 [[TMP1]], ptr addrspace(10) @_ZL12ushort_const, align 2
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.5(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantij(i32 6, i32 12)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10uint_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.6(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i64 @_Z20__spirv_SpecConstantiy(i32 7, i64 25)
+// CHECK-NEXT:    store i64 [[TMP1]], ptr addrspace(10) @_ZL11ulong_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.7(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantiDh(i32 8, float 0x4044333340000000)
+// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL10half_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.8(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn float @_Z20__spirv_SpecConstantif(i32 8, float 5.050000e+01)
+// CHECK-NEXT:    store float [[TMP1]], ptr addrspace(10) @_ZL11float_const, align 4
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.9(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call reassoc nnan ninf nsz arcp afn double @_Z20__spirv_SpecConstantid(i32 9, double 0x40590CCCC0000000)
+// CHECK-NEXT:    store double [[TMP1]], ptr addrspace(10) @_ZL12double_const, align 8
+// CHECK-NEXT:    ret void
+//
+//
+// CHECK-LABEL: define internal spir_func void @__cxx_global_var_init.10(
+// CHECK-SAME: ) #[[ATTR3]] {
+// CHECK-NEXT:  [[ENTRY:.*:]]
+// CHECK-NEXT:    [[TMP0:%.*]] = call token @llvm.experimental.convergence.entry()
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @_Z20__spirv_SpecConstantii(i32 10, i32 30)
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(10) @_ZL10enum_const, align 4
+// CHECK-NEXT:    ret void
diff --git a/clang/test/SemaHLSL/vk.spec-constant.error.hlsl b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
new file mode 100644
index 000000000000..24873d272a54
--- /dev/null
+++ b/clang/test/SemaHLSL/vk.spec-constant.error.hlsl
@@ -0,0 +1,37 @@
+// RUN: %clang_cc1 -finclude-default-header -triple spirv-pc-vulkan1.3-compute -verify %s
+// RUN: %clang_cc1 -finclude-default-header -triple dxil-pc-shadermodel6.8-compute -verify %s
+
+#ifndef __spirv__
+// expected-warning@+2{{'constant_id' attribute ignored}}
+#endif
+[[vk::constant_id(0)]]
+const bool sc0 = true;
+
+#ifdef __spirv__
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(1)]]
+const bool sc1 = sc0; // error
+
+// expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+[[vk::constant_id(2)]]
+static const bool sc2 = false; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(3)]]
+const bool sc3; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(4)]]
+bool sc4 = false; // error
+
+// expected-error@+2{{variable with 'vk::constant_id' attribute must be a const int/float/enum/bool and be initialized with a literal}}
+[[vk::constant_id(5)]]
+const int2 sc5 = {0,0}; // error
+
+[numthreads(1,1,1)]
+void main() {
+  // expected-warning@+1{{'constant_id' attribute only applies to external global variables}}
+  [[vk::constant_id(6)]]
+  const bool sc6 = false; // error
+}
+#endif
-- 
cgit v1.2.3


From c1ac87b327861a7387c1ab9e1ffb1c002acbcd6a Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Thu, 19 Jun 2025 12:05:58 -0400
Subject: [libc] Fix setjmp build order. (#144917)

Fix build order issue from
https://github.com/llvm/llvm-project/pull/139555.
---
 libc/src/setjmp/CMakeLists.txt | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libc/src/setjmp/CMakeLists.txt b/libc/src/setjmp/CMakeLists.txt
index 50c827254da6..8b8e74f0955e 100644
--- a/libc/src/setjmp/CMakeLists.txt
+++ b/libc/src/setjmp/CMakeLists.txt
@@ -1,7 +1,4 @@
 # Process architecture-specific subdirectory FIRST to avoid missing targets.
-if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
-endif()
 
 # Then process OS-specific subdirectory
 if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
@@ -14,6 +11,10 @@ if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS})
   )
 endif()
 
+if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_ARCHITECTURE})
+endif()
+
 add_entrypoint_object(
   setjmp
   ALIAS
-- 
cgit v1.2.3


From b8337349d9b6143669e8bfa6776926a708cacf99 Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Thu, 19 Jun 2025 12:18:39 -0400
Subject: [libc][math] Skip setting errno and floating point exception for math
 functions when LIBC_MATH flag has LIBC_MATH_NO_ERRNO and LIBC_MATH_NO_EXCEPT.
 (#144920)

---
 libc/src/__support/FPUtil/FEnvImpl.h     | 17 +++++++++++++----
 libc/src/__support/macros/optimization.h |  8 ++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/libc/src/__support/FPUtil/FEnvImpl.h b/libc/src/__support/FPUtil/FEnvImpl.h
index 50a101f833c5..ba145a3da45c 100644
--- a/libc/src/__support/FPUtil/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/FEnvImpl.h
@@ -15,6 +15,7 @@
 #include "src/__support/libc_errno.h"
 #include "src/__support/macros/attributes.h" // LIBC_INLINE
 #include "src/__support/macros/config.h"
+#include "src/__support/macros/optimization.h"
 #include "src/__support/macros/properties/architectures.h"
 
 #if defined(LIBC_TARGET_ARCH_IS_AARCH64) && defined(__ARM_FP)
@@ -71,27 +72,35 @@ LIBC_INLINE int set_env(const fenv_t *) { return 0; }
 namespace LIBC_NAMESPACE_DECL {
 namespace fputil {
 
-LIBC_INLINE int clear_except_if_required(int excepts) {
+LIBC_INLINE static int clear_except_if_required([[maybe_unused]] int excepts) {
+#ifndef LIBC_MATH_HAS_NO_EXCEPT
   if (math_errhandling & MATH_ERREXCEPT)
     return clear_except(excepts);
+#endif // LIBC_MATH_HAS_NO_EXCEPT
   return 0;
 }
 
-LIBC_INLINE int set_except_if_required(int excepts) {
+LIBC_INLINE static int set_except_if_required([[maybe_unused]] int excepts) {
+#ifndef LIBC_MATH_HAS_NO_EXCEPT
   if (math_errhandling & MATH_ERREXCEPT)
     return set_except(excepts);
+#endif // LIBC_MATH_HAS_NO_EXCEPT
   return 0;
 }
 
-LIBC_INLINE int raise_except_if_required(int excepts) {
+LIBC_INLINE static int raise_except_if_required([[maybe_unused]] int excepts) {
+#ifndef LIBC_MATH_HAS_NO_EXCEPT
   if (math_errhandling & MATH_ERREXCEPT)
     return raise_except(excepts);
+#endif // LIBC_MATH_HAS_NO_EXCEPT
   return 0;
 }
 
-LIBC_INLINE void set_errno_if_required(int err) {
+LIBC_INLINE static void set_errno_if_required([[maybe_unused]] int err) {
+#ifndef LIBC_MATH_HAS_NO_ERRNO
   if (math_errhandling & MATH_ERRNO)
     libc_errno = err;
+#endif // LIBC_MATH_HAS_NO_ERRNO
 }
 
 } // namespace fputil
diff --git a/libc/src/__support/macros/optimization.h b/libc/src/__support/macros/optimization.h
index 253843e5e37a..db008d323b3a 100644
--- a/libc/src/__support/macros/optimization.h
+++ b/libc/src/__support/macros/optimization.h
@@ -63,4 +63,12 @@ LIBC_INLINE constexpr bool expects_bool_condition(T value, T expected) {
 #define LIBC_MATH_HAS_INTERMEDIATE_COMP_IN_FLOAT
 #endif
 
+#if (LIBC_MATH & LIBC_MATH_NO_ERRNO)
+#define LIBC_MATH_HAS_NO_ERRNO
+#endif
+
+#if (LIBC_MATH & LIBC_MATH_NO_EXCEPT)
+#define LIBC_MATH_HAS_NO_EXCEPT
+#endif
+
 #endif // LLVM_LIBC_SRC___SUPPORT_MACROS_OPTIMIZATION_H
-- 
cgit v1.2.3


From 5cf7d871b030212d021ffc9356620551f09ad402 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Thu, 19 Jun 2025 09:25:04 -0700
Subject: [NFC][Clang][AST] Adopt simplified `getTrailingObjects` in AST
 (#144432)

Adopt simplified `getTrailingObjects` API in several places in clag/AST
that were missed by earlier changes.
---
 clang/include/clang/AST/DeclTemplate.h  | 16 ++++---------
 clang/include/clang/AST/Expr.h          |  7 ++----
 clang/include/clang/AST/OpenACCClause.h |  2 +-
 clang/include/clang/AST/OpenMPClause.h  | 42 +++++++++------------------------
 clang/include/clang/AST/StmtOpenACC.h   |  2 +-
 clang/include/clang/AST/Type.h          |  6 ++---
 clang/lib/AST/ASTImporter.cpp           |  2 +-
 clang/lib/AST/Decl.cpp                  |  2 +-
 clang/lib/AST/ExprCXX.cpp               |  6 ++---
 clang/lib/AST/OpenMPClause.cpp          | 12 +++++-----
 clang/lib/AST/Type.cpp                  |  2 +-
 11 files changed, 33 insertions(+), 66 deletions(-)

diff --git a/clang/include/clang/AST/DeclTemplate.h b/clang/include/clang/AST/DeclTemplate.h
index 8d8b1ca93882..939b14b0351d 100644
--- a/clang/include/clang/AST/DeclTemplate.h
+++ b/clang/include/clang/AST/DeclTemplate.h
@@ -279,7 +279,7 @@ public:
 
   /// Produce this as an array ref.
   ArrayRef<TemplateArgument> asArray() const {
-    return llvm::ArrayRef(data(), size());
+    return getTrailingObjects(size());
   }
 
   /// Retrieve the number of template arguments in this
@@ -287,9 +287,7 @@ public:
   unsigned size() const { return NumArguments; }
 
   /// Retrieve a pointer to the template argument list.
-  const TemplateArgument *data() const {
-    return getTrailingObjects<TemplateArgument>();
-  }
+  const TemplateArgument *data() const { return getTrailingObjects(); }
 };
 
 void *allocateDefaultArgStorageChain(const ASTContext &C);
@@ -505,12 +503,10 @@ private:
         TemplateArgumentsAsWritten(TemplateArgsAsWritten),
         PointOfInstantiation(POI) {
     if (MSInfo)
-      getTrailingObjects<MemberSpecializationInfo *>()[0] = MSInfo;
+      getTrailingObjects()[0] = MSInfo;
   }
 
-  size_t numTrailingObjects(OverloadToken<MemberSpecializationInfo*>) const {
-    return Function.getInt();
-  }
+  size_t numTrailingObjects() const { return Function.getInt(); }
 
 public:
   friend TrailingObjects;
@@ -597,9 +593,7 @@ public:
   /// function and the function template, and should always be
   /// TSK_ExplicitSpecialization whenever we have MemberSpecializationInfo.
   MemberSpecializationInfo *getMemberSpecializationInfo() const {
-    return numTrailingObjects(OverloadToken<MemberSpecializationInfo *>())
-               ? getTrailingObjects<MemberSpecializationInfo *>()[0]
-               : nullptr;
+    return numTrailingObjects() ? getTrailingObjects()[0] : nullptr;
   }
 
   void Profile(llvm::FoldingSetNodeID &ID) {
diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h
index 9fc23d30b733..41e50359962e 100644
--- a/clang/include/clang/AST/Expr.h
+++ b/clang/include/clang/AST/Expr.h
@@ -7364,17 +7364,14 @@ public:
                               ArrayRef<Expr *> SubExprs);
   static RecoveryExpr *CreateEmpty(ASTContext &Ctx, unsigned NumSubExprs);
 
-  ArrayRef<Expr *> subExpressions() {
-    auto *B = getTrailingObjects<Expr *>();
-    return llvm::ArrayRef(B, B + NumExprs);
-  }
+  ArrayRef<Expr *> subExpressions() { return getTrailingObjects(NumExprs); }
 
   ArrayRef<const Expr *> subExpressions() const {
     return const_cast<RecoveryExpr *>(this)->subExpressions();
   }
 
   child_range children() {
-    Stmt **B = reinterpret_cast<Stmt **>(getTrailingObjects<Expr *>());
+    Stmt **B = reinterpret_cast<Stmt **>(getTrailingObjects());
     return child_range(B, B + NumExprs);
   }
 
diff --git a/clang/include/clang/AST/OpenACCClause.h b/clang/include/clang/AST/OpenACCClause.h
index 67fbdfeb0702..a778c7cc3dc9 100644
--- a/clang/include/clang/AST/OpenACCClause.h
+++ b/clang/include/clang/AST/OpenACCClause.h
@@ -307,7 +307,7 @@ public:
   }
 
   ArrayRef<DeviceTypeArgument> getArchitectures() const {
-    return getTrailingObjects<DeviceTypeArgument>(NumArchs);
+    return getTrailingObjects(NumArchs);
   }
 
   static OpenACCDeviceTypeClause *
diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 6fd16bc0f03b..2fa8fa529741 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -302,8 +302,7 @@ protected:
   void setVarRefs(ArrayRef<Expr *> VL) {
     assert(VL.size() == NumVars &&
            "Number of variables is not the same as the preallocated buffer");
-    std::copy(VL.begin(), VL.end(),
-              static_cast<T *>(this)->template getTrailingObjects<Expr *>());
+    llvm::copy(VL, getVarRefs().begin());
   }
 
 public:
@@ -388,9 +387,7 @@ public:
     assert(
         DK.size() == NumKinds &&
         "Number of directive kinds is not the same as the preallocated buffer");
-    std::copy(DK.begin(), DK.end(),
-              static_cast<T *>(this)
-                  ->template getTrailingObjects<OpenMPDirectiveKind>());
+    std::copy(DK.begin(), DK.end(), getDirectiveKinds().begin());
   }
 
   SourceLocation getLParenLoc() { return LParenLoc; }
@@ -980,20 +977,14 @@ public:
 
   /// Returns the tile size expressions.
   MutableArrayRef<Expr *> getSizesRefs() {
-    return static_cast<OMPSizesClause *>(this)
-        ->template getTrailingObjects<Expr *>(NumSizes);
-  }
-  ArrayRef<Expr *> getSizesRefs() const {
-    return static_cast<const OMPSizesClause *>(this)
-        ->template getTrailingObjects<Expr *>(NumSizes);
+    return getTrailingObjects(NumSizes);
   }
+  ArrayRef<Expr *> getSizesRefs() const { return getTrailingObjects(NumSizes); }
 
   /// Sets the tile size expressions.
   void setSizesRefs(ArrayRef<Expr *> VL) {
     assert(VL.size() == NumSizes);
-    std::copy(VL.begin(), VL.end(),
-              static_cast<OMPSizesClause *>(this)
-                  ->template getTrailingObjects<Expr *>());
+    llvm::copy(VL, getSizesRefs().begin());
   }
 
   child_range children() {
@@ -1043,8 +1034,7 @@ class OMPPermutationClause final
   /// Sets the permutation index expressions.
   void setArgRefs(ArrayRef<Expr *> VL) {
     assert(VL.size() == NumLoops && "Expecting one expression per loop");
-    llvm::copy(VL, static_cast<OMPPermutationClause *>(this)
-                       ->template getTrailingObjects<Expr *>());
+    llvm::copy(VL, getTrailingObjects());
   }
 
   /// Build an empty clause.
@@ -1083,14 +1073,8 @@ public:
 
   /// Returns the permutation index expressions.
   ///@{
-  MutableArrayRef<Expr *> getArgsRefs() {
-    return static_cast<OMPPermutationClause *>(this)
-        ->template getTrailingObjects<Expr *>(NumLoops);
-  }
-  ArrayRef<Expr *> getArgsRefs() const {
-    return static_cast<const OMPPermutationClause *>(this)
-        ->template getTrailingObjects<Expr *>(NumLoops);
-  }
+  MutableArrayRef<Expr *> getArgsRefs() { return getTrailingObjects(NumLoops); }
+  ArrayRef<Expr *> getArgsRefs() const { return getTrailingObjects(NumLoops); }
   ///@}
 
   child_range children() {
@@ -9239,9 +9223,7 @@ class OMPAffinityClause final
                                             SourceLocation(), N) {}
 
   /// Sets the affinity modifier for the clause, if any.
-  void setModifier(Expr *E) {
-    getTrailingObjects<Expr *>()[varlist_size()] = E;
-  }
+  void setModifier(Expr *E) { getTrailingObjects()[varlist_size()] = E; }
 
   /// Sets the location of ':' symbol.
   void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
@@ -9268,10 +9250,8 @@ public:
   static OMPAffinityClause *CreateEmpty(const ASTContext &C, unsigned N);
 
   /// Gets affinity modifier.
-  Expr *getModifier() { return getTrailingObjects<Expr *>()[varlist_size()]; }
-  Expr *getModifier() const {
-    return getTrailingObjects<Expr *>()[varlist_size()];
-  }
+  Expr *getModifier() { return getTrailingObjects()[varlist_size()]; }
+  Expr *getModifier() const { return getTrailingObjects()[varlist_size()]; }
 
   /// Gets the location of ':' symbol.
   SourceLocation getColonLoc() const { return ColonLoc; }
diff --git a/clang/include/clang/AST/StmtOpenACC.h b/clang/include/clang/AST/StmtOpenACC.h
index c8f8b968b1c8..9ad3d8e00d98 100644
--- a/clang/include/clang/AST/StmtOpenACC.h
+++ b/clang/include/clang/AST/StmtOpenACC.h
@@ -736,7 +736,7 @@ class OpenACCUpdateConstruct final
                              OpenACCDirectiveKind::Update, SourceLocation{},
                              SourceLocation{}, SourceLocation{}) {
     std::uninitialized_value_construct_n(getTrailingObjects(), NumClauses);
-    setClauseList(getTrailingObjects<const OpenACCClause *>(NumClauses));
+    setClauseList(getTrailingObjects(NumClauses));
   }
 
   OpenACCUpdateConstruct(SourceLocation Start, SourceLocation DirectiveLoc,
diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h
index 3896cd914bf0..35a8b898d8e1 100644
--- a/clang/include/clang/AST/Type.h
+++ b/clang/include/clang/AST/Type.h
@@ -6052,9 +6052,7 @@ public:
                       ArrayRef<QualType> Expansions);
 
 private:
-  const QualType *getExpansionsPtr() const {
-    return getTrailingObjects<QualType>();
-  }
+  const QualType *getExpansionsPtr() const { return getTrailingObjects(); }
 
   static TypeDependence computeDependence(QualType Pattern, Expr *IndexExpr,
                                           ArrayRef<QualType> Expansions = {});
@@ -6494,7 +6492,7 @@ public:
   uint32_t getSize() const { return Size; }
   uint32_t getAlignment() const { return Alignment; }
   ArrayRef<SpirvOperand> getOperands() const {
-    return getTrailingObjects<SpirvOperand>(NumOperands);
+    return getTrailingObjects(NumOperands);
   }
 
   bool isSugared() const { return false; }
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 5c44353d8b98..96a5e2eeaa4d 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -4452,7 +4452,7 @@ ExpectedDecl ASTNodeImporter::VisitFriendDecl(FriendDecl *D) {
   }
 
   SmallVector<TemplateParameterList *, 1> ToTPLists(D->NumTPLists);
-  auto **FromTPLists = D->getTrailingObjects<TemplateParameterList *>();
+  auto **FromTPLists = D->getTrailingObjects();
   for (unsigned I = 0; I < D->NumTPLists; I++) {
     if (auto ListOrErr = import(FromTPLists[I]))
       ToTPLists[I] = *ListOrErr;
diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp
index 860968939b4a..c4376aab480c 100644
--- a/clang/lib/AST/Decl.cpp
+++ b/clang/lib/AST/Decl.cpp
@@ -5380,7 +5380,7 @@ PragmaCommentDecl *PragmaCommentDecl::Create(const ASTContext &C,
       new (C, DC, additionalSizeToAlloc<char>(Arg.size() + 1))
           PragmaCommentDecl(DC, CommentLoc, CommentKind);
   memcpy(PCD->getTrailingObjects(), Arg.data(), Arg.size());
-  PCD->getTrailingObjects<char>()[Arg.size()] = '\0';
+  PCD->getTrailingObjects()[Arg.size()] = '\0';
   return PCD;
 }
 
diff --git a/clang/lib/AST/ExprCXX.cpp b/clang/lib/AST/ExprCXX.cpp
index 43b1c39d7379..2b66445fe253 100644
--- a/clang/lib/AST/ExprCXX.cpp
+++ b/clang/lib/AST/ExprCXX.cpp
@@ -806,8 +806,7 @@ CXXDynamicCastExpr *CXXDynamicCastExpr::Create(const ASTContext &C, QualType T,
       new (Buffer) CXXDynamicCastExpr(T, VK, K, Op, PathSize, WrittenTy, L,
                                       RParenLoc, AngleBrackets);
   if (PathSize)
-    llvm::uninitialized_copy(*BasePath,
-                             E->getTrailingObjects<CXXBaseSpecifier *>());
+    llvm::uninitialized_copy(*BasePath, E->getTrailingObjects());
   return E;
 }
 
@@ -869,8 +868,7 @@ CXXReinterpretCastExpr::Create(const ASTContext &C, QualType T,
       new (Buffer) CXXReinterpretCastExpr(T, VK, K, Op, PathSize, WrittenTy, L,
                                           RParenLoc, AngleBrackets);
   if (PathSize)
-    llvm::uninitialized_copy(*BasePath,
-                             E->getTrailingObjects<CXXBaseSpecifier *>());
+    llvm::uninitialized_copy(*BasePath, E->getTrailingObjects());
   return E;
 }
 
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 0e5052b94416..f714974b9476 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -370,26 +370,26 @@ OMPOrderedClause *OMPOrderedClause::CreateEmpty(const ASTContext &C,
 void OMPOrderedClause::setLoopNumIterations(unsigned NumLoop,
                                             Expr *NumIterations) {
   assert(NumLoop < NumberOfLoops && "out of loops number.");
-  getTrailingObjects<Expr *>()[NumLoop] = NumIterations;
+  getTrailingObjects()[NumLoop] = NumIterations;
 }
 
 ArrayRef<Expr *> OMPOrderedClause::getLoopNumIterations() const {
-  return getTrailingObjects<Expr *>(NumberOfLoops);
+  return getTrailingObjects(NumberOfLoops);
 }
 
 void OMPOrderedClause::setLoopCounter(unsigned NumLoop, Expr *Counter) {
   assert(NumLoop < NumberOfLoops && "out of loops number.");
-  getTrailingObjects<Expr *>()[NumberOfLoops + NumLoop] = Counter;
+  getTrailingObjects()[NumberOfLoops + NumLoop] = Counter;
 }
 
 Expr *OMPOrderedClause::getLoopCounter(unsigned NumLoop) {
   assert(NumLoop < NumberOfLoops && "out of loops number.");
-  return getTrailingObjects<Expr *>()[NumberOfLoops + NumLoop];
+  return getTrailingObjects()[NumberOfLoops + NumLoop];
 }
 
 const Expr *OMPOrderedClause::getLoopCounter(unsigned NumLoop) const {
   assert(NumLoop < NumberOfLoops && "out of loops number.");
-  return getTrailingObjects<Expr *>()[NumberOfLoops + NumLoop];
+  return getTrailingObjects()[NumberOfLoops + NumLoop];
 }
 
 OMPUpdateClause *OMPUpdateClause::Create(const ASTContext &C,
@@ -1678,7 +1678,7 @@ OMPInitClause *OMPInitClause::Create(const ASTContext &C, Expr *InteropVar,
       InteropInfo.IsTarget, InteropInfo.IsTargetSync, StartLoc, LParenLoc,
       VarLoc, EndLoc, InteropInfo.PreferTypes.size() + 1);
   Clause->setInteropVar(InteropVar);
-  llvm::copy(InteropInfo.PreferTypes, Clause->getTrailingObjects<Expr *>() + 1);
+  llvm::copy(InteropInfo.PreferTypes, Clause->getTrailingObjects() + 1);
   return Clause;
 }
 
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
index a461dbde4093..543f05e4ee7c 100644
--- a/clang/lib/AST/Type.cpp
+++ b/clang/lib/AST/Type.cpp
@@ -3981,7 +3981,7 @@ CountAttributedType::CountAttributedType(
   CountAttributedTypeBits.NumCoupledDecls = CoupledDecls.size();
   CountAttributedTypeBits.CountInBytes = CountInBytes;
   CountAttributedTypeBits.OrNull = OrNull;
-  auto *DeclSlot = getTrailingObjects<TypeCoupledDeclRefInfo>();
+  auto *DeclSlot = getTrailingObjects();
   Decls = llvm::ArrayRef(DeclSlot, CoupledDecls.size());
   for (unsigned i = 0; i != CoupledDecls.size(); ++i)
     DeclSlot[i] = CoupledDecls[i];
-- 
cgit v1.2.3


From c0cc81cdc03c97473ba771bbc3a2330bd22396bc Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Thu, 19 Jun 2025 09:25:32 -0700
Subject: [NFC][Clang] Adopt simplified `getTrailingObjects` in ASTReader
 (#144438)

---
 clang/lib/Serialization/ASTReaderDecl.cpp | 24 +++++++++++-------------
 clang/lib/Serialization/ASTReaderStmt.cpp | 19 +++++++++----------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp
index 259c772e4222..7f7882654b9d 100644
--- a/clang/lib/Serialization/ASTReaderDecl.cpp
+++ b/clang/lib/Serialization/ASTReaderDecl.cpp
@@ -664,22 +664,21 @@ void ASTDeclReader::VisitPragmaCommentDecl(PragmaCommentDecl *D) {
   D->setLocation(readSourceLocation());
   D->CommentKind = (PragmaMSCommentKind)Record.readInt();
   std::string Arg = readString();
-  memcpy(D->getTrailingObjects<char>(), Arg.data(), Arg.size());
-  D->getTrailingObjects<char>()[Arg.size()] = '\0';
+  memcpy(D->getTrailingObjects(), Arg.data(), Arg.size());
+  D->getTrailingObjects()[Arg.size()] = '\0';
 }
 
 void ASTDeclReader::VisitPragmaDetectMismatchDecl(PragmaDetectMismatchDecl *D) {
   VisitDecl(D);
   D->setLocation(readSourceLocation());
   std::string Name = readString();
-  memcpy(D->getTrailingObjects<char>(), Name.data(), Name.size());
-  D->getTrailingObjects<char>()[Name.size()] = '\0';
+  memcpy(D->getTrailingObjects(), Name.data(), Name.size());
+  D->getTrailingObjects()[Name.size()] = '\0';
 
   D->ValueStart = Name.size() + 1;
   std::string Value = readString();
-  memcpy(D->getTrailingObjects<char>() + D->ValueStart, Value.data(),
-         Value.size());
-  D->getTrailingObjects<char>()[D->ValueStart + Value.size()] = '\0';
+  memcpy(D->getTrailingObjects() + D->ValueStart, Value.data(), Value.size());
+  D->getTrailingObjects()[D->ValueStart + Value.size()] = '\0';
 }
 
 void ASTDeclReader::VisitTranslationUnitDecl(TranslationUnitDecl *TU) {
@@ -1746,7 +1745,7 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) {
 
 void ASTDeclReader::VisitDecompositionDecl(DecompositionDecl *DD) {
   VisitVarDecl(DD);
-  auto **BDs = DD->getTrailingObjects<BindingDecl *>();
+  auto **BDs = DD->getTrailingObjects();
   for (unsigned I = 0; I != DD->NumBindings; ++I) {
     BDs[I] = readDeclAs<BindingDecl>();
     BDs[I]->setDecomposedDecl(DD);
@@ -1917,7 +1916,7 @@ void ASTDeclReader::VisitUsingEnumDecl(UsingEnumDecl *D) {
 void ASTDeclReader::VisitUsingPackDecl(UsingPackDecl *D) {
   VisitNamedDecl(D);
   D->InstantiatedFrom = readDeclAs<NamedDecl>();
-  auto **Expansions = D->getTrailingObjects<NamedDecl *>();
+  auto **Expansions = D->getTrailingObjects();
   for (unsigned I = 0; I != D->NumExpansions; ++I)
     Expansions[I] = readDeclAs<NamedDecl>();
   mergeMergeable(D);
@@ -2358,7 +2357,7 @@ void ASTDeclReader::VisitImportDecl(ImportDecl *D) {
   VisitDecl(D);
   D->ImportedModule = readModule();
   D->setImportComplete(Record.readInt());
-  auto *StoredLocs = D->getTrailingObjects<SourceLocation>();
+  auto *StoredLocs = D->getTrailingObjects();
   for (unsigned I = 0, N = Record.back(); I != N; ++I)
     StoredLocs[I] = readSourceLocation();
   Record.skipInts(1); // The number of stored source locations.
@@ -2376,8 +2375,7 @@ void ASTDeclReader::VisitFriendDecl(FriendDecl *D) {
   else
     D->Friend = readTypeSourceInfo();
   for (unsigned i = 0; i != D->NumTPLists; ++i)
-    D->getTrailingObjects<TemplateParameterList *>()[i] =
-        Record.readTemplateParameterList();
+    D->getTrailingObjects()[i] = Record.readTemplateParameterList();
   D->NextFriend = readDeclID().getRawValue();
   D->UnsupportedFriend = (Record.readInt() != 0);
   D->FriendLoc = readSourceLocation();
@@ -2745,7 +2743,7 @@ void ASTDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   D->setDepth(Record.readInt());
   D->setPosition(Record.readInt());
   if (D->isExpandedParameterPack()) {
-    auto **Data = D->getTrailingObjects<TemplateParameterList *>();
+    auto **Data = D->getTrailingObjects();
     for (unsigned I = 0, N = D->getNumExpansionTemplateParameters();
          I != N; ++I)
       Data[I] = Record.readTemplateParameterList();
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 44cfb83ad2db..8945407cf666 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -719,7 +719,7 @@ void ASTStmtReader::VisitParenListExpr(ParenListExpr *E) {
   unsigned NumExprs = Record.readInt();
   assert((NumExprs == E->getNumExprs()) && "Wrong NumExprs!");
   for (unsigned I = 0; I != NumExprs; ++I)
-    E->getTrailingObjects<Stmt *>()[I] = Record.readSubStmt();
+    E->getTrailingObjects()[I] = Record.readSubStmt();
   E->LParenLoc = readSourceLocation();
   E->RParenLoc = readSourceLocation();
 }
@@ -1892,7 +1892,7 @@ void ASTStmtReader::VisitCXXDefaultArgExpr(CXXDefaultArgExpr *E) {
   E->CXXDefaultArgExprBits.Loc = readSourceLocation();
   E->CXXDefaultArgExprBits.HasRewrittenInit = Record.readInt();
   if (E->CXXDefaultArgExprBits.HasRewrittenInit)
-    *E->getTrailingObjects<Expr *>() = Record.readSubExpr();
+    *E->getTrailingObjects() = Record.readSubExpr();
 }
 
 void ASTStmtReader::VisitCXXDefaultInitExpr(CXXDefaultInitExpr *E) {
@@ -1902,7 +1902,7 @@ void ASTStmtReader::VisitCXXDefaultInitExpr(CXXDefaultInitExpr *E) {
   E->UsedContext = readDeclAs<DeclContext>();
   E->CXXDefaultInitExprBits.Loc = readSourceLocation();
   if (E->CXXDefaultInitExprBits.HasRewrittenInit)
-    *E->getTrailingObjects<Expr *>() = Record.readSubExpr();
+    *E->getTrailingObjects() = Record.readSubExpr();
 }
 
 void ASTStmtReader::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *E) {
@@ -1999,7 +1999,7 @@ void ASTStmtReader::VisitExprWithCleanups(ExprWithCleanups *E) {
       Obj = cast<CompoundLiteralExpr>(Record.readSubExpr());
     else
       llvm_unreachable("unexpected cleanup object type");
-    E->getTrailingObjects<ExprWithCleanups::CleanupObject>()[i] = Obj;
+    E->getTrailingObjects()[i] = Obj;
   }
 
   E->ExprWithCleanupsBits.CleanupsHaveSideEffects = Record.readInt();
@@ -2198,9 +2198,8 @@ void ASTStmtReader::VisitSizeOfPackExpr(SizeOfPackExpr *E) {
   E->Pack = Record.readDeclAs<NamedDecl>();
   if (E->isPartiallySubstituted()) {
     assert(E->Length == NumPartialArgs);
-    for (auto *I = E->getTrailingObjects<TemplateArgument>(),
-              *E = I + NumPartialArgs;
-         I != E; ++I)
+    for (auto *I = E->getTrailingObjects(), *E = I + NumPartialArgs; I != E;
+         ++I)
       new (I) TemplateArgument(Record.readTemplateArgument());
   } else if (!E->isValueDependent()) {
     E->Length = Record.readInt();
@@ -2215,7 +2214,7 @@ void ASTStmtReader::VisitPackIndexingExpr(PackIndexingExpr *E) {
   E->RSquareLoc = readSourceLocation();
   E->SubExprs[0] = Record.readStmt();
   E->SubExprs[1] = Record.readStmt();
-  auto **Exprs = E->getTrailingObjects<Expr *>();
+  auto **Exprs = E->getTrailingObjects();
   for (unsigned I = 0; I < E->PackIndexingExprBits.TransformedExpressions; ++I)
     Exprs[I] = Record.readExpr();
 }
@@ -2252,7 +2251,7 @@ void ASTStmtReader::VisitFunctionParmPackExpr(FunctionParmPackExpr *E) {
   E->NumParameters = Record.readInt();
   E->ParamPack = readDeclAs<ValueDecl>();
   E->NameLoc = readSourceLocation();
-  auto **Parms = E->getTrailingObjects<ValueDecl *>();
+  auto **Parms = E->getTrailingObjects();
   for (unsigned i = 0, n = E->NumParameters; i != n; ++i)
     Parms[i] = readDeclAs<ValueDecl>();
 }
@@ -2289,7 +2288,7 @@ void ASTStmtReader::VisitCXXParenListInitExpr(CXXParenListInitExpr *E) {
   E->LParenLoc = readSourceLocation();
   E->RParenLoc = readSourceLocation();
   for (unsigned I = 0; I < ExpectedNumExprs; I++)
-    E->getTrailingObjects<Expr *>()[I] = Record.readSubExpr();
+    E->getTrailingObjects()[I] = Record.readSubExpr();
 
   bool HasArrayFillerOrUnionDecl = Record.readBool();
   if (HasArrayFillerOrUnionDecl) {
-- 
cgit v1.2.3


From 3fe62682ef9ca514b899d0cecaebb8f1fd97baef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Thu, 19 Jun 2025 17:34:08 +0100
Subject: [mlir][vector] Use `result` consistently as the result argument name
 (#144739)

This patch updates the following ops to use `result` (instead of `res`)
as the name for their result argument:
  * `vector.scalable.insert`
  * `vector.scalable.extract`
  * `vector.insert_strided_slice`

This change ensures naming consistency with other ops in the `vector`
dialect. It addresses part of:
* https://github.com/llvm/llvm-project/issues/131602
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td | 28 +++++++++++++++++-------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 125cd4645ccc..85cc22ab3964 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1008,7 +1008,7 @@ def Vector_InsertOp :
 def Vector_ScalableInsertOp :
   Vector_Op<"scalable.insert", [Pure,
        AllElementTypesMatch<["valueToStore", "dest"]>,
-       AllTypesMatch<["dest", "res"]>,
+       AllTypesMatch<["dest", "result"]>,
        PredOpTrait<"position is a multiple of the source length.",
         CPred<
           "(getPos() % getSourceVectorType().getNumElements()) == 0"
@@ -1016,7 +1016,7 @@ def Vector_ScalableInsertOp :
      Arguments<(ins VectorOfRank<[1]>:$valueToStore,
                     ScalableVectorOfRank<[1]>:$dest,
                     I64Attr:$pos)>,
-     Results<(outs ScalableVectorOfRank<[1]>:$res)> {
+     Results<(outs ScalableVectorOfRank<[1]>:$result)> {
   let summary = "insert subvector into scalable vector operation";
   // NOTE: This operation is designed to map to `llvm.vector.insert`, and its
   //       documentation should be kept aligned with LLVM IR:
@@ -1059,19 +1059,23 @@ def Vector_ScalableInsertOp :
     VectorType getDestVectorType() {
       return ::llvm::cast<VectorType>(getDest().getType());
     }
+    /// Wrapper for getResult, which replaced getRes.
+    [[deprecated("Use getResult instead!")]] ::mlir::Value getRes() {
+      return getResult();
+    }
   }];
 }
 
 def Vector_ScalableExtractOp :
   Vector_Op<"scalable.extract", [Pure,
-       AllElementTypesMatch<["source", "res"]>,
+       AllElementTypesMatch<["source", "result"]>,
        PredOpTrait<"position is a multiple of the result length.",
         CPred<
           "(getPos() % getResultVectorType().getNumElements()) == 0"
         >>]>,
      Arguments<(ins ScalableVectorOfRank<[1]>:$source,
                     I64Attr:$pos)>,
-     Results<(outs VectorOfRank<[1]>:$res)> {
+     Results<(outs VectorOfRank<[1]>:$result)> {
   let summary = "extract subvector from scalable vector operation";
   // NOTE: This operation is designed to map to `llvm.vector.extract`, and its
   //       documentation should be kept aligned with LLVM IR:
@@ -1100,7 +1104,7 @@ def Vector_ScalableExtractOp :
   }];
 
   let assemblyFormat = [{
-    $source `[` $pos `]` attr-dict `:` type($res) `from` type($source)
+    $source `[` $pos `]` attr-dict `:` type($result) `from` type($source)
   }];
 
   let extraClassDeclaration = extraPoisonClassDeclaration # [{
@@ -1108,7 +1112,11 @@ def Vector_ScalableExtractOp :
       return ::llvm::cast<VectorType>(getSource().getType());
     }
     VectorType getResultVectorType() {
-      return ::llvm::cast<VectorType>(getRes().getType());
+      return ::llvm::cast<VectorType>(getResult().getType());
+    }
+    /// Wrapper for getResult, which replaced getRes.
+    [[deprecated("Use getResult instead!")]] ::mlir::Value getRes() {
+      return getResult();
     }
   }];
 }
@@ -1117,10 +1125,10 @@ def Vector_InsertStridedSliceOp :
   Vector_Op<"insert_strided_slice", [Pure,
     PredOpTrait<"operand #0 and result have same element type",
                  TCresVTEtIsSameAsOpBase<0, 0>>,
-    AllTypesMatch<["dest", "res"]>]>,
+    AllTypesMatch<["dest", "result"]>]>,
     Arguments<(ins AnyVectorOfNonZeroRank:$valueToStore, AnyVectorOfNonZeroRank:$dest, I64ArrayAttr:$offsets,
                I64ArrayAttr:$strides)>,
-    Results<(outs AnyVectorOfNonZeroRank:$res)> {
+    Results<(outs AnyVectorOfNonZeroRank:$result)> {
   let summary = "strided_slice operation";
   let description = [{
     Takes a k-D valueToStore vector, an n-D destination vector (n >= k), n-sized
@@ -1164,6 +1172,10 @@ def Vector_InsertStridedSliceOp :
         return ::llvm::cast<IntegerAttr>(attr).getInt() != 1;
       });
     }
+    /// Wrapper for getResult, which replaced getRes.
+    [[deprecated("Use getResult instead!")]] ::mlir::Value getRes() {
+      return getResult();
+    }
   }];
 
   let hasFolder = 1;
-- 
cgit v1.2.3


From 0816bb32ac37b24d2f895f0c0464b7659fffd4fc Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 19 Jun 2025 17:49:33 +0100
Subject: [Matrix] Fix heap-use-after-free after 0fa373c77ded203eddb.

We need to skip instructions in FusedInsts, as they may have been
deleted. Fixes a heap-use-after-free after #141681.
---
 llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index fa9e44617b7c..ccb68700747b 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -1140,6 +1140,9 @@ public:
     // Fourth, pre-process all the PHINode's. The incoming values will be
     // assigned later in VisitPHI.
     for (Instruction *Inst : MatrixInsts) {
+      if (FusedInsts.count(Inst))
+        continue;
+
       auto *PHI = dyn_cast<PHINode>(Inst);
       if (!PHI)
         continue;
-- 
cgit v1.2.3


From 6ce86538c11b3ef93a2a8df3bd4f817a724f42bd Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 19 Jun 2025 10:09:10 -0700
Subject: [mlir][cf] Preserve branch weights during cf.cond_br
 canonicalization. (#144822)

---
 .../mlir/Dialect/ControlFlow/IR/ControlFlowOps.td  | 20 +++++++++++------
 mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp |  6 +++---
 mlir/test/Dialect/ControlFlow/canonicalize.mlir    | 25 ++++++++++++++++++++++
 3 files changed, 42 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td b/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
index 79da81ba049d..a441fd82546e 100644
--- a/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
+++ b/mlir/include/mlir/Dialect/ControlFlow/IR/ControlFlowOps.td
@@ -153,17 +153,25 @@ def CondBranchOp
   let builders = [OpBuilder<(ins "Value":$condition, "Block *":$trueDest,
                                 "ValueRange":$trueOperands,
                                 "Block *":$falseDest,
-                                "ValueRange":$falseOperands),
+                                "ValueRange":$falseOperands,
+                                CArg<"ArrayRef<int32_t>", "{}">:$branchWeights),
                             [{
-      build($_builder, $_state, condition, trueOperands, falseOperands, /*branch_weights=*/{}, trueDest,
-            falseDest);
+      DenseI32ArrayAttr weights;
+      if (!branchWeights.empty())
+        weights = $_builder.getDenseI32ArrayAttr(branchWeights);
+      build($_builder, $_state, condition, trueOperands, falseOperands,
+            weights, trueDest, falseDest);
     }]>,
                   OpBuilder<(ins "Value":$condition, "Block *":$trueDest,
                                 "Block *":$falseDest,
-                                CArg<"ValueRange", "{}">:$falseOperands),
+                                CArg<"ValueRange", "{}">:$falseOperands,
+                                CArg<"ArrayRef<int32_t>", "{}">:$branchWeights),
                             [{
-      build($_builder, $_state, condition, trueDest, ValueRange(), falseDest,
-            falseOperands);
+      DenseI32ArrayAttr weights;
+      if (!branchWeights.empty())
+        weights = $_builder.getDenseI32ArrayAttr(branchWeights);
+      build($_builder, $_state, condition, ValueRange(), falseOperands,
+            weights, trueDest, falseDest);
     }]>];
 
   let extraClassDeclaration = [{
diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
index edd7f607f24f..0c11c76cf1f7 100644
--- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
+++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp
@@ -265,9 +265,9 @@ struct SimplifyPassThroughCondBranch : public OpRewritePattern<CondBranchOp> {
       return failure();
 
     // Create a new branch with the collapsed successors.
-    rewriter.replaceOpWithNewOp<CondBranchOp>(condbr, condbr.getCondition(),
-                                              trueDest, trueDestOperands,
-                                              falseDest, falseDestOperands);
+    rewriter.replaceOpWithNewOp<CondBranchOp>(
+        condbr, condbr.getCondition(), trueDest, trueDestOperands, falseDest,
+        falseDestOperands, condbr.getWeights());
     return success();
   }
 };
diff --git a/mlir/test/Dialect/ControlFlow/canonicalize.mlir b/mlir/test/Dialect/ControlFlow/canonicalize.mlir
index 0ad6898fce86..bf69935a00bf 100644
--- a/mlir/test/Dialect/ControlFlow/canonicalize.mlir
+++ b/mlir/test/Dialect/ControlFlow/canonicalize.mlir
@@ -102,6 +102,31 @@ func.func @cond_br_and_br_folding(%a : i32) {
 
 /// Test that pass-through successors of CondBranchOp get folded.
 
+// Test that the weights are preserved:
+// CHECK-LABEL:   func.func @cond_br_passthrough_weights(
+// CHECK-SAME:      %[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i1) -> i32 {
+func.func @cond_br_passthrough_weights(%arg0 : i32, %arg1 : i32, %cond : i1) -> i32 {
+// CHECK:           cf.cond_br %[[ARG2]] weights([30, 70]), ^bb1, ^bb2
+// CHECK:         ^bb1:
+// CHECK:           return %[[ARG0]] : i32
+// CHECK:         ^bb2:
+// CHECK:           return %[[ARG1]] : i32
+// CHECK:         }
+  cf.cond_br %cond weights([30,70]), ^bb1, ^bb3
+
+^bb1:
+  cf.br ^bb2
+
+^bb3:
+  cf.br ^bb4
+
+^bb2:
+  return %arg0 : i32
+
+^bb4:
+  return %arg1 : i32
+}
+
 // CHECK-LABEL: func @cond_br_passthrough(
 // CHECK-SAME: %[[ARG0:.*]]: i32, %[[ARG1:.*]]: i32, %[[ARG2:.*]]: i32, %[[COND:.*]]: i1
 func.func @cond_br_passthrough(%arg0 : i32, %arg1 : i32, %arg2 : i32, %cond : i1) -> (i32, i32) {
-- 
cgit v1.2.3


From c0c71463f6bca05eb4540b68cdcbd17c916562c9 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 19 Jun 2025 10:13:58 -0700
Subject: [InstCombine] Optimize sub(sext(add(x,y)),sext(add(x,z))). (#144174)

This pattern can be often met in Flang generated LLVM IR,
for example, for the counts of the loops generated for array
expressions like: `a(x:x+y)` or `a(x+z:x+z)` or their variations.

In order to compute the loop count, Flang needs to subtract
the lower bound of the array slice from the upper bound
of the array slice. To avoid the sign wraps, it sign extends
the original values (that may be of any user data type)
to `i64`.

This peephole is really helpful in CPU2017/548.exchange2,
where we have multiple following statements like this:
```
block(row+1:row+2, 7:9, i7) = block(row+1:row+2, 7:9, i7) - 10
```

While this is just a 2x3 iterations loop nest, LLVM cannot
figure it out, ending up vectorizing the inner loop really
hard (with a vector epilog and scalar remainder). This, in turn,
causes problems for LSR that ends up creating too many loop-carried
values in the loop containing the above statement, which are then
causing too many spills/reloads.

Alive2: https://alive2.llvm.org/ce/z/gLgfYX

Related to #143219.
---
 llvm/include/llvm/IR/PatternMatch.h                |   8 +
 .../Transforms/InstCombine/InstCombineAddSub.cpp   |  51 ++-
 .../InstCombine/InstCombineMulDivRem.cpp           |   6 +-
 .../Transforms/InstCombine/InstCombineNegator.cpp  |   8 +-
 llvm/test/Transforms/InstCombine/sub-sext-add.ll   | 350 +++++++++++++++++++++
 5 files changed, 413 insertions(+), 10 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/sub-sext-add.ll

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 2eaa7d0faabc..1f86cdfd94e1 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -1323,6 +1323,14 @@ m_NSWAdd(const LHS &L, const RHS &R) {
                                                                             R);
 }
 template <typename LHS, typename RHS>
+inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
+                                 OverflowingBinaryOperator::NoSignedWrap, true>
+m_c_NSWAdd(const LHS &L, const RHS &R) {
+  return OverflowingBinaryOp_match<LHS, RHS, Instruction::Add,
+                                   OverflowingBinaryOperator::NoSignedWrap,
+                                   true>(L, R);
+}
+template <typename LHS, typename RHS>
 inline OverflowingBinaryOp_match<LHS, RHS, Instruction::Sub,
                                  OverflowingBinaryOperator::NoSignedWrap>
 m_NSWSub(const LHS &L, const RHS &R) {
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 0a3837f2c0ce..418302d1edb3 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -1896,7 +1896,7 @@ Instruction *InstCombinerImpl::visitAdd(BinaryOperator &I) {
                                           {Sub, Builder.getFalse()});
     Value *Ret = Builder.CreateSub(
         ConstantInt::get(A->getType(), A->getType()->getScalarSizeInBits()),
-        Ctlz, "", /*HasNUW*/ true, /*HasNSW*/ true);
+        Ctlz, "", /*HasNUW=*/true, /*HasNSW=*/true);
     return replaceInstUsesWith(I, Builder.CreateZExtOrTrunc(Ret, I.getType()));
   }
 
@@ -2363,8 +2363,8 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
     OverflowingBinaryOperator *LHSSub = cast<OverflowingBinaryOperator>(Op0);
     bool HasNUW = I.hasNoUnsignedWrap() && LHSSub->hasNoUnsignedWrap();
     bool HasNSW = HasNUW && I.hasNoSignedWrap() && LHSSub->hasNoSignedWrap();
-    Value *Add = Builder.CreateAdd(Y, Op1, "", /* HasNUW */ HasNUW,
-                                   /* HasNSW */ HasNSW);
+    Value *Add = Builder.CreateAdd(Y, Op1, "", /*HasNUW=*/HasNUW,
+                                   /*HasNSW=*/HasNSW);
     BinaryOperator *Sub = BinaryOperator::CreateSub(X, Add);
     Sub->setHasNoUnsignedWrap(HasNUW);
     Sub->setHasNoSignedWrap(HasNSW);
@@ -2835,6 +2835,51 @@ Instruction *InstCombinerImpl::visitSub(BinaryOperator &I) {
   if (Instruction *Res = foldBinOpOfSelectAndCastOfSelectCondition(I))
     return Res;
 
+  // (sub (sext (add nsw (X, Y)), sext (X))) --> (sext (Y))
+  if (match(Op1, m_SExtLike(m_Value(X))) &&
+      match(Op0, m_SExtLike(m_c_NSWAdd(m_Specific(X), m_Value(Y))))) {
+    Value *SExtY = Builder.CreateSExt(Y, I.getType());
+    return replaceInstUsesWith(I, SExtY);
+  }
+
+  // (sub[ nsw] (sext (add nsw (X, Y)), sext (add nsw (X, Z)))) -->
+  // --> (sub[ nsw] (sext (Y), sext (Z)))
+  {
+    Value *Z, *Add0, *Add1;
+    if (match(Op0, m_SExtLike(m_Value(Add0))) &&
+        match(Op1, m_SExtLike(m_Value(Add1))) &&
+        ((match(Add0, m_NSWAdd(m_Value(X), m_Value(Y))) &&
+          match(Add1, m_c_NSWAdd(m_Specific(X), m_Value(Z)))) ||
+         (match(Add0, m_NSWAdd(m_Value(Y), m_Value(X))) &&
+          match(Add1, m_c_NSWAdd(m_Specific(X), m_Value(Z)))))) {
+      unsigned NumOfNewInstrs = 0;
+      // Non-constant Y, Z require new SExt.
+      NumOfNewInstrs += !isa<Constant>(Y) ? 1 : 0;
+      NumOfNewInstrs += !isa<Constant>(Z) ? 1 : 0;
+      // Check if we can trade some of the old instructions for the new ones.
+      unsigned NumOfDeadInstrs = 0;
+      if (Op0->hasOneUse()) {
+        // If Op0 (sext) has multiple uses, then we keep it
+        // and the add that it uses, otherwise, we can remove
+        // the sext and probably the add (depending on the number of its uses).
+        ++NumOfDeadInstrs;
+        NumOfDeadInstrs += Add0->hasOneUse() ? 1 : 0;
+      }
+      if (Op1->hasOneUse()) {
+        ++NumOfDeadInstrs;
+        NumOfDeadInstrs += Add1->hasOneUse() ? 1 : 0;
+      }
+      if (NumOfDeadInstrs >= NumOfNewInstrs) {
+        Value *SExtY = Builder.CreateSExt(Y, I.getType());
+        Value *SExtZ = Builder.CreateSExt(Z, I.getType());
+        Value *Sub = Builder.CreateSub(SExtY, SExtZ, "",
+                                       /*HasNUW=*/false,
+                                       /*HasNSW=*/I.hasNoSignedWrap());
+        return replaceInstUsesWith(I, Sub);
+      }
+    }
+  }
+
   return TryToNarrowDeduceFlags();
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index fcf4613b5d13..d7310b1c741c 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -290,7 +290,7 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) {
       auto *Op1C = cast<Constant>(Op1);
       return replaceInstUsesWith(
           I, Builder.CreateMul(NegOp0, ConstantExpr::getNeg(Op1C), "",
-                               /* HasNUW */ false,
+                               /*HasNUW=*/false,
                                HasNSW && Op1C->isNotMinSignedValue()));
     }
 
@@ -1255,8 +1255,8 @@ static Value *foldIDivShl(BinaryOperator &I, InstCombiner::BuilderTy &Builder) {
       // or divisor has nsw and operator is sdiv.
       Value *Dividend = Builder.CreateShl(
           One, Y, "shl.dividend",
-          /*HasNUW*/ true,
-          /*HasNSW*/
+          /*HasNUW=*/true,
+          /*HasNSW=*/
           IsSigned ? (Shl0->hasNoUnsignedWrap() || Shl1->hasNoUnsignedWrap())
                    : Shl0->hasNoSignedWrap());
       return Builder.CreateLShr(Dividend, Z, "", I.isExact());
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
index 2210336d92bf..b0a0bcfbde19 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp
@@ -233,7 +233,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
     // However, only do this either if the old `sub` doesn't stick around, or
     // it was subtracting from a constant. Otherwise, this isn't profitable.
     return Builder.CreateSub(I->getOperand(1), I->getOperand(0),
-                             I->getName() + ".neg", /* HasNUW */ false,
+                             I->getName() + ".neg", /*HasNUW=*/false,
                              IsNSW && I->hasNoSignedWrap());
   }
 
@@ -404,7 +404,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
     IsNSW &= I->hasNoSignedWrap();
     if (Value *NegOp0 = negate(I->getOperand(0), IsNSW, Depth + 1))
       return Builder.CreateShl(NegOp0, I->getOperand(1), I->getName() + ".neg",
-                               /* HasNUW */ false, IsNSW);
+                               /*HasNUW=*/false, IsNSW);
     // Otherwise, `shl %x, C` can be interpreted as `mul %x, 1<<C`.
     Constant *Op1C;
     if (!match(I->getOperand(1), m_ImmConstant(Op1C)) || !IsTrulyNegation)
@@ -412,7 +412,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
     return Builder.CreateMul(
         I->getOperand(0),
         Builder.CreateShl(Constant::getAllOnesValue(Op1C->getType()), Op1C),
-        I->getName() + ".neg", /* HasNUW */ false, IsNSW);
+        I->getName() + ".neg", /*HasNUW=*/false, IsNSW);
   }
   case Instruction::Or: {
     if (!cast<PossiblyDisjointInst>(I)->isDisjoint())
@@ -483,7 +483,7 @@ std::array<Value *, 2> Negator::getSortedOperandsOfBinOp(Instruction *I) {
       // Can't negate either of them.
       return nullptr;
     return Builder.CreateMul(NegatedOp, OtherOp, I->getName() + ".neg",
-                             /* HasNUW */ false, IsNSW && I->hasNoSignedWrap());
+                             /*HasNUW=*/false, IsNSW && I->hasNoSignedWrap());
   }
   default:
     return nullptr; // Don't know, likely not negatible for free.
diff --git a/llvm/test/Transforms/InstCombine/sub-sext-add.ll b/llvm/test/Transforms/InstCombine/sub-sext-add.ll
new file mode 100644
index 000000000000..71e91f954420
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/sub-sext-add.ll
@@ -0,0 +1,350 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define i64 @src_2add_2sext_sub_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_1(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_2(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %z, %x
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_3(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %y, %x
+  %add2 = add nsw i32 %z, %x
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_4(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %y, %x
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sextlike_sub(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sextlike_sub(
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = zext nneg i32 %add1 to i64
+  %sext2 = zext nneg i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_nsw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_nsw(
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub nsw i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_nuw(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_nuw(
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub nuw i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+declare void @use_i32(i32, i32)
+declare void @use_i64(i64, i64)
+
+define i64 @src_2add_2sext_sub_multiple_uses_1(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_1(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT1]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i64(i64 %sext1, i64 %sext1)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_2(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_2(
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT2]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i64(i64 %sext2, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_3(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_3(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i64(i64 %sext1, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_4(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_4(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD1]], i32 [[ADD2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add1, i32 %add2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_5(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_5(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z:%.*]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD1]], i32 [[ADD1]])
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT1]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add1, i32 %add1)
+  call void @use_i64(i64 %sext1, i64 %sext1)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_6(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_6(
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i32 [[Z]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[TMP1]], [[TMP2]]
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD2]], i32 [[ADD2]])
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT2]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add2, i32 %add2)
+  call void @use_i64(i64 %sext2, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_7(i32 %x, i32 %y, i32 %z) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_7(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X]], [[Z:%.*]]
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD1]], i32 [[ADD2]])
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, %z
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add1, i32 %add2)
+  call void @use_i64(i64 %sext1, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_8(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_8(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], [[Y:%.*]]
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    [[SUB:%.*]] = sub nsw i64 [[SEXT1]], [[SEXT2]]
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD1]], i32 [[ADD2]])
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %add2 = add nsw i32 %x, 1
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add1, i32 %add2)
+  call void @use_i64(i64 %sext1, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_2add_2sext_sub_multiple_uses_9(i32 %x) {
+; CHECK-LABEL: @src_2add_2sext_sub_multiple_uses_9(
+; CHECK-NEXT:    [[ADD1:%.*]] = add nsw i32 [[X:%.*]], 2
+; CHECK-NEXT:    [[ADD2:%.*]] = add nsw i32 [[X]], 1
+; CHECK-NEXT:    [[SEXT1:%.*]] = sext i32 [[ADD1]] to i64
+; CHECK-NEXT:    [[SEXT2:%.*]] = sext i32 [[ADD2]] to i64
+; CHECK-NEXT:    call void @use_i32(i32 [[ADD1]], i32 [[ADD2]])
+; CHECK-NEXT:    call void @use_i64(i64 [[SEXT1]], i64 [[SEXT2]])
+; CHECK-NEXT:    ret i64 1
+;
+  %add1 = add nsw i32 %x, 2
+  %add2 = add nsw i32 %x, 1
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %add2 to i64
+  %sub = sub i64 %sext1, %sext2
+  call void @use_i32(i32 %add1, i32 %add2)
+  call void @use_i64(i64 %sext1, i64 %sext2)
+  ret i64 %sub
+}
+
+define i64 @src_x_add_2sext_sub_1(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_x_add_2sext_sub_1(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %x to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_x_add_2sext_sub_2(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_x_add_2sext_sub_2(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %y, %x
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %x to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_x_add_2sextlike_sub(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_x_add_2sextlike_sub(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %sext1 = zext nneg i32 %add1 to i64
+  %sext2 = zext nneg i32 %x to i64
+  %sub = sub i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_x_add_2sext_sub_nsw(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_x_add_2sext_sub_nsw(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %x to i64
+  %sub = sub nsw i64 %sext1, %sext2
+  ret i64 %sub
+}
+
+define i64 @src_x_add_2sext_sub_nuw(i32 %x, i32 %y) {
+; CHECK-LABEL: @src_x_add_2sext_sub_nuw(
+; CHECK-NEXT:    [[SUB:%.*]] = sext i32 [[Y:%.*]] to i64
+; CHECK-NEXT:    ret i64 [[SUB]]
+;
+  %add1 = add nsw i32 %x, %y
+  %sext1 = sext i32 %add1 to i64
+  %sext2 = sext i32 %x to i64
+  %sub = sub nuw i64 %sext1, %sext2
+  ret i64 %sub
+}
-- 
cgit v1.2.3


From 0b8179b2adbc821324c425d7cafd269f84e72d5e Mon Sep 17 00:00:00 2001
From: Abhishek Kaushik <abhishek.kaushik@intel.com>
Date: Thu, 19 Jun 2025 10:35:40 -0700
Subject: [ValueTracking] Improve `Bitcast` handling to match SDAG (#125935)

Closes #125228
---
 llvm/lib/Analysis/ValueTracking.cpp                | 27 ++++++++++++++++++++--
 .../InstCombine/X86/x86-vector-shifts.ll           |  4 ++--
 .../Transforms/InstCombine/bitcast-known-bits.ll   | 21 ++++++-----------
 3 files changed, 34 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index a17417cb5189..73320b556f82 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -1346,6 +1346,8 @@ static void computeKnownBitsFromOperator(const Operator *I,
         isa<ScalableVectorType>(I->getType()))
       break;
 
+    unsigned NumElts = DemandedElts.getBitWidth();
+    bool IsLE = Q.DL.isLittleEndian();
     // Look through a cast from narrow vector elements to wider type.
     // Examples: v4i32 -> v2i64, v3i8 -> v24
     unsigned SubBitWidth = SrcVecTy->getScalarSizeInBits();
@@ -1364,7 +1366,6 @@ static void computeKnownBitsFromOperator(const Operator *I,
       //
       // The known bits of each sub-element are then inserted into place
       // (dependent on endian) to form the full result of known bits.
-      unsigned NumElts = DemandedElts.getBitWidth();
       unsigned SubScale = BitWidth / SubBitWidth;
       APInt SubDemandedElts = APInt::getZero(NumElts * SubScale);
       for (unsigned i = 0; i != NumElts; ++i) {
@@ -1376,10 +1377,32 @@ static void computeKnownBitsFromOperator(const Operator *I,
       for (unsigned i = 0; i != SubScale; ++i) {
         computeKnownBits(I->getOperand(0), SubDemandedElts.shl(i), KnownSrc, Q,
                          Depth + 1);
-        unsigned ShiftElt = Q.DL.isLittleEndian() ? i : SubScale - 1 - i;
+        unsigned ShiftElt = IsLE ? i : SubScale - 1 - i;
         Known.insertBits(KnownSrc, ShiftElt * SubBitWidth);
       }
     }
+    // Look through a cast from wider vector elements to narrow type.
+    // Examples: v2i64 -> v4i32
+    if (SubBitWidth % BitWidth == 0) {
+      unsigned SubScale = SubBitWidth / BitWidth;
+      KnownBits KnownSrc(SubBitWidth);
+      APInt SubDemandedElts =
+          APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
+      computeKnownBits(I->getOperand(0), SubDemandedElts, KnownSrc, Q,
+                       Depth + 1);
+
+      Known.Zero.setAllBits();
+      Known.One.setAllBits();
+      for (unsigned i = 0; i != SubScale; ++i) {
+        if (DemandedElts[i]) {
+          unsigned Shifts = IsLE ? i : NumElts - 1 - i;
+          unsigned Offset = (Shifts % SubScale) * BitWidth;
+          Known = Known.intersectWith(KnownSrc.extractBits(BitWidth, Offset));
+          if (Known.isUnknown())
+            break;
+        }
+      }
+    }
     break;
   }
   case Instruction::SExt: {
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
index db56080a3ea2..cc252ae53803 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
@@ -3732,7 +3732,6 @@ define <4 x i64> @test_avx2_psrl_0() {
   ret <4 x i64> %16
 }
 
-; FIXME: Failure to peek through bitcasts to ensure psllq shift amount is within bounds.
 define <2 x i64> @PR125228(<2 x i64> %v, <2 x i64> %s) {
 ; CHECK-LABEL: @PR125228(
 ; CHECK-NEXT:    [[MASK:%.*]] = and <2 x i64> [[S:%.*]], splat (i64 63)
@@ -3741,7 +3740,8 @@ define <2 x i64> @PR125228(<2 x i64> %v, <2 x i64> %s) {
 ; CHECK-NEXT:    [[CAST:%.*]] = bitcast <2 x i64> [[MASK]] to <16 x i8>
 ; CHECK-NEXT:    [[PSRLDQ:%.*]] = shufflevector <16 x i8> [[CAST]], <16 x i8> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
 ; CHECK-NEXT:    [[CAST3:%.*]] = bitcast <16 x i8> [[PSRLDQ]] to <2 x i64>
-; CHECK-NEXT:    [[SLL1:%.*]] = call <2 x i64> @llvm.x86.sse2.psll.q(<2 x i64> [[V]], <2 x i64> [[CAST3]])
+; CHECK-NEXT:    [[TMP2:%.*]] = shufflevector <2 x i64> [[CAST3]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[SLL1:%.*]] = shl <2 x i64> [[V]], [[TMP2]]
 ; CHECK-NEXT:    [[SHUFP_UNCASTED:%.*]] = shufflevector <2 x i64> [[SLL0]], <2 x i64> [[SLL1]], <2 x i32> <i32 0, i32 3>
 ; CHECK-NEXT:    ret <2 x i64> [[SHUFP_UNCASTED]]
 ;
diff --git a/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll b/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll
index 3e47e775e3a2..65b43df752f7 100644
--- a/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll
+++ b/llvm/test/Transforms/InstCombine/bitcast-known-bits.ll
@@ -12,8 +12,7 @@ define <16 x i8> @knownbits_bitcast_masked_shift(<16 x i8> %arg1, <16 x i8> %arg
 ; CHECK-NEXT:    [[BITCAST4:%.*]] = bitcast <16 x i8> [[OR]] to <8 x i16>
 ; CHECK-NEXT:    [[SHL5:%.*]] = shl nuw <8 x i16> [[BITCAST4]], splat (i16 2)
 ; CHECK-NEXT:    [[BITCAST6:%.*]] = bitcast <8 x i16> [[SHL5]] to <16 x i8>
-; CHECK-NEXT:    [[AND7:%.*]] = and <16 x i8> [[BITCAST6]], splat (i8 -52)
-; CHECK-NEXT:    ret <16 x i8> [[AND7]]
+; CHECK-NEXT:    ret <16 x i8> [[BITCAST6]]
 ;
   %and = and <16 x i8> %arg1, splat (i8 3)
   %and3 = and <16 x i8> %arg2, splat (i8 48)
@@ -33,8 +32,7 @@ define <16 x i8> @knownbits_shuffle_masked_nibble_shift(<16 x i8> %arg)  {
 ; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <16 x i8> [[SHUFFLEVECTOR]] to <8 x i16>
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <8 x i16> [[BITCAST1]], splat (i16 4)
 ; CHECK-NEXT:    [[BITCAST2:%.*]] = bitcast <8 x i16> [[SHL]] to <16 x i8>
-; CHECK-NEXT:    [[AND3:%.*]] = and <16 x i8> [[BITCAST2]], splat (i8 -16)
-; CHECK-NEXT:    ret <16 x i8> [[AND3]]
+; CHECK-NEXT:    ret <16 x i8> [[BITCAST2]]
 ;
   %and = and <16 x i8> %arg, splat (i8 15)
   %shufflevector = shufflevector <16 x i8> %and, <16 x i8> poison, <16 x i32> <i32 1, i32 0, i32 3, i32 2, i32 5, i32 4, i32 7, i32 6, i32 9, i32 8, i32 11, i32 10, i32 13, i32 12, i32 15, i32 14>
@@ -53,8 +51,7 @@ define <16 x i8> @knownbits_reverse_shuffle_masked_shift(<16 x i8> %arg)  {
 ; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <16 x i8> [[SHUFFLEVECTOR]] to <8 x i16>
 ; CHECK-NEXT:    [[SHL:%.*]] = shl nuw <8 x i16> [[BITCAST1]], splat (i16 4)
 ; CHECK-NEXT:    [[BITCAST2:%.*]] = bitcast <8 x i16> [[SHL]] to <16 x i8>
-; CHECK-NEXT:    [[AND3:%.*]] = and <16 x i8> [[BITCAST2]], splat (i8 -16)
-; CHECK-NEXT:    ret <16 x i8> [[AND3]]
+; CHECK-NEXT:    ret <16 x i8> [[BITCAST2]]
 ;
   %and = and <16 x i8> %arg, splat (i8 15)
   %shufflevector = shufflevector <16 x i8> %and, <16 x i8> poison, <16 x i32> <i32 3, i32 2, i32 1, i32 0, i32 7, i32 6, i32 5, i32 4, i32 11, i32 10, i32 9, i32 8, i32 15, i32 14, i32 13, i32 12>
@@ -70,8 +67,7 @@ define <16 x i8> @knownbits_extract_bit(<8 x i16> %arg)  {
 ; CHECK-SAME: <8 x i16> [[ARG:%.*]]) {
 ; CHECK-NEXT:    [[LSHR:%.*]] = lshr <8 x i16> [[ARG]], splat (i16 15)
 ; CHECK-NEXT:    [[BITCAST1:%.*]] = bitcast <8 x i16> [[LSHR]] to <16 x i8>
-; CHECK-NEXT:    [[AND:%.*]] = and <16 x i8> [[BITCAST1]], splat (i8 1)
-; CHECK-NEXT:    ret <16 x i8> [[AND]]
+; CHECK-NEXT:    ret <16 x i8> [[BITCAST1]]
 ;
   %lshr = lshr <8 x i16> %arg, splat (i16 15)
   %bitcast1 = bitcast <8 x i16> %lshr to <16 x i8>
@@ -88,7 +84,8 @@ define { i32, i1 } @knownbits_popcount_add_with_overflow(<2 x i64> %arg1, <2 x i
 ; CHECK-NEXT:    [[CALL9:%.*]] = tail call range(i64 0, 65) <2 x i64> @llvm.ctpop.v2i64(<2 x i64> [[ARG2]])
 ; CHECK-NEXT:    [[BITCAST10:%.*]] = bitcast <2 x i64> [[CALL9]] to <4 x i32>
 ; CHECK-NEXT:    [[EXTRACTELEMENT11:%.*]] = extractelement <4 x i32> [[BITCAST10]], i64 0
-; CHECK-NEXT:    [[TMP1:%.*]] = tail call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 [[EXTRACTELEMENT]], i32 [[EXTRACTELEMENT11]])
+; CHECK-NEXT:    [[CALL12:%.*]] = add nuw nsw i32 [[EXTRACTELEMENT]], [[EXTRACTELEMENT11]]
+; CHECK-NEXT:    [[TMP1:%.*]] = insertvalue { i32, i1 } { i32 poison, i1 false }, i32 [[CALL12]], 0
 ; CHECK-NEXT:    ret { i32, i1 } [[TMP1]]
 ;
   %call = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %arg1)
@@ -110,11 +107,7 @@ define <16 x i8> @knownbits_shuffle_add_shift_v32i8(<16 x i8> %arg1, <8 x i16> %
 ; CHECK-NEXT:    [[BITCAST11:%.*]] = bitcast <8 x i16> [[SHL10]] to <16 x i8>
 ; CHECK-NEXT:    [[ADD12:%.*]] = add <16 x i8> [[BITCAST11]], [[BITCAST7]]
 ; CHECK-NEXT:    [[ADD14:%.*]] = add <16 x i8> [[ADD12]], [[ARG1]]
-; CHECK-NEXT:    [[BITCAST14:%.*]] = bitcast <16 x i8> [[ADD12]] to <8 x i16>
-; CHECK-NEXT:    [[SHL15:%.*]] = shl <8 x i16> [[BITCAST14]], splat (i16 8)
-; CHECK-NEXT:    [[BITCAST16:%.*]] = bitcast <8 x i16> [[SHL15]] to <16 x i8>
-; CHECK-NEXT:    [[ADD13:%.*]] = add <16 x i8> [[ADD14]], [[BITCAST16]]
-; CHECK-NEXT:    ret <16 x i8> [[ADD13]]
+; CHECK-NEXT:    ret <16 x i8> [[ADD14]]
 ;
   %shl6 = shl <8 x i16> %arg2, splat (i16 8)
   %bitcast7 = bitcast <8 x i16> %shl6 to <16 x i8>
-- 
cgit v1.2.3


From f780955e1df9105e9c4e67ebd16efded7dd279e2 Mon Sep 17 00:00:00 2001
From: Justin King <jcking@google.com>
Date: Thu, 19 Jun 2025 10:38:45 -0700
Subject: lsan: fix macos build after #144604 (#144818)

Fixes build failures on macOS, including

https://green.lab.llvm.org/job/llvm.org/job/clang-stage1-RA/

llvm-project/compiler-rt/lib/lsan/lsan_interceptors.cpp:579:3: error: use of undeclared identifier 'LSAN_MAYBE_INTERCEPT_FREE_SIZED'
13:23:58    579 |   LSAN_MAYBE_INTERCEPT_FREE_SIZED;
13:23:58        |   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
13:23:58  /Users/ec2-user/jenkins/workspace/llvm.org/as-lldb-cmake/llvm-project/compiler-rt/lib/lsan/lsan_interceptors.cpp:580:3: error: use of undeclared identifier 'LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED'
13:23:58    580 |   LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED;
13:23:58        |   ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
13:23:58  2 errors generated.
---
 compiler-rt/lib/lsan/lsan_interceptors.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/compiler-rt/lib/lsan/lsan_interceptors.cpp b/compiler-rt/lib/lsan/lsan_interceptors.cpp
index 6da9d0d9d24f..f9f83f6c0cc4 100644
--- a/compiler-rt/lib/lsan/lsan_interceptors.cpp
+++ b/compiler-rt/lib/lsan/lsan_interceptors.cpp
@@ -146,6 +146,9 @@ INTERCEPTOR(void*, valloc, uptr size) {
   GET_STACK_TRACE_MALLOC;
   return lsan_valloc(size, stack);
 }
+#else
+#  define LSAN_MAYBE_INTERCEPT_FREE_SIZED
+#  define LSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
 #endif  // !SANITIZER_APPLE
 
 #if SANITIZER_INTERCEPT_MEMALIGN
-- 
cgit v1.2.3


From 836201f1177c38f3ca0457de019bb179a04afe3c Mon Sep 17 00:00:00 2001
From: Umang Yadav <29876643+umangyadav@users.noreply.github.com>
Date: Thu, 19 Jun 2025 13:52:31 -0400
Subject: Allow bf16 operands on new MFMAs  (#144925)

New gfx950 MFMA allows bf16 operands.


https://github.com/llvm/llvm-project/blob/c0cc81cdc03c97473ba771bbc3a2330bd22396bc/llvm/include/llvm/IR/IntrinsicsAMDGPU.td#L3434

When running `amdgpu-to-rocdl`, Current logic converts bf16 to i16
always which fails to compile for newer bf16 MFMA e.g.
`v_mfma_f32_16x16x32bf16`.
Backend expects bf16 type for the operands for those newer MFMAs. This
patch fixes it.

CC: @krzysz00  @dhernandez0  @giuseros  @antiagainst  @kuhar
---
 .../lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp | 28 ++++++++++++++++------
 .../test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir |  4 ++--
 2 files changed, 23 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
index 074404add47f..700563460f52 100644
--- a/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
+++ b/mlir/lib/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.cpp
@@ -499,7 +499,9 @@ struct SchedBarrierOpLowering : public ConvertOpToLLVMPattern<SchedBarrierOp> {
 /// and LLVM AMDGPU intrinsics convention.
 ///
 /// Specifically:
-/// 1. If the element type is bfloat16, bitcast it to i16.
+/// 1. If the element type is bfloat16, bitcast it to i16 unless rocdl intrinsic
+/// allows bf16. Newer MFMAs support bf16 types on operand, check
+/// IntrinsicsAMDGPU.td file for reference.
 /// 2. If instead we have a more than 64-bit quantity, use a <N / 4 x i32>
 /// instead, which is what the f8f6f4 intrinsics use.
 /// 3. If `input` is a vector of N <= 8 bytes, bitcast it to a (N * 8)-bit
@@ -509,10 +511,11 @@ struct SchedBarrierOpLowering : public ConvertOpToLLVMPattern<SchedBarrierOp> {
 /// therefore 8-bit and smaller floats are represented as their corresponding
 /// `iN` integers.
 static Value convertMFMAVectorOperand(ConversionPatternRewriter &rewriter,
-                                      Location loc, Value input) {
+                                      Location loc, Value input,
+                                      bool allowBf16 = true) {
   Type inputType = input.getType();
   if (auto vectorType = dyn_cast<VectorType>(inputType)) {
-    if (vectorType.getElementType().isBF16())
+    if (vectorType.getElementType().isBF16() && !allowBf16)
       return rewriter.create<LLVM::BitcastOp>(
           loc, vectorType.clone(rewriter.getI16Type()), input);
     if (vectorType.getElementType().isInteger(8) &&
@@ -958,12 +961,23 @@ struct MFMAOpLowering : public ConvertOpToLLVMPattern<MFMAOp> {
 
     StringRef intrinsicName =
         isScaled ? std::get<0>(*maybeScaledIntrinsic) : *maybeIntrinsic;
+    // Determine if we can use bf16 in the intrinsic. Newer MFMAs in gfx950+
+    // allows bf16 as the input. For reference check IntrinsicsAMDGPU.td file.
+    bool allowBf16 = [&]() {
+      if (chipset < kGfx950)
+        return false;
+      if (isScaled)
+        return true;
+      return intrinsicName.contains("16x16x32.bf16") ||
+             intrinsicName.contains("32x32x16.bf16");
+    }();
     OperationState loweredOp(loc, intrinsicName);
     loweredOp.addTypes(intrinsicOutType);
-    loweredOp.addOperands(
-        {convertMFMAVectorOperand(rewriter, loc, adaptor.getSourceA()),
-         convertMFMAVectorOperand(rewriter, loc, adaptor.getSourceB()),
-         adaptor.getDestC()});
+    loweredOp.addOperands({convertMFMAVectorOperand(
+                               rewriter, loc, adaptor.getSourceA(), allowBf16),
+                           convertMFMAVectorOperand(
+                               rewriter, loc, adaptor.getSourceB(), allowBf16),
+                           adaptor.getDestC()});
     if (isScaled) {
       Value zero = createI32Constant(rewriter, loc, 0);
       auto [_scaledName, aTypeCode, bTypeCode] = *maybeScaledIntrinsic;
diff --git a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
index 52a5d39f668c..39c31d5bf2fa 100644
--- a/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
+++ b/mlir/test/Conversion/AMDGPUToROCDL/mfma-gfx950.mlir
@@ -11,9 +11,9 @@ func.func @mfma_to_rocdl(%arg0 : vector<8xf16>, %arg1 : vector<16xf32>,
   amdgpu.mfma %arg0 * %arg0 + %arg1 { abid = 0 : i32, cbsz = 0 : i32, k = 16 : i32, m = 32 : i32, n = 32 : i32, blocks = 1 : i32 }  blgp = none : vector<8xf16>, vector<8xf16>, vector<16xf32>
   // CHECK: rocdl.mfma.f32.16x16x32.f16{{.*}}: (vector<8xf16>, vector<8xf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
   amdgpu.mfma %arg0 * %arg0 + %arg2 { abid = 0 : i32, cbsz = 0 : i32, k = 32 : i32, m = 16 : i32, n = 16 : i32, blocks = 1 : i32 }  blgp = none : vector<8xf16>, vector<8xf16>, vector<4xf32>
-  // CHECK: rocdl.mfma.f32.32x32x16.bf16{{.*}}: (vector<8xi16>, vector<8xi16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
+  // CHECK: rocdl.mfma.f32.32x32x16.bf16{{.*}}: (vector<8xbf16>, vector<8xbf16>, vector<16xf32>, i32, i32, i32) -> vector<16xf32>
   amdgpu.mfma %arg3 * %arg3 + %arg1 { abid = 0 : i32, cbsz = 0 : i32, k = 16 : i32, m = 32 : i32, n = 32 : i32, blocks = 1 : i32 }  blgp = none : vector<8xbf16>, vector<8xbf16>, vector<16xf32>
-  // CHECK: rocdl.mfma.f32.16x16x32.bf16{{.*}}: (vector<8xi16>, vector<8xi16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
+  // CHECK: rocdl.mfma.f32.16x16x32.bf16{{.*}}: (vector<8xbf16>, vector<8xbf16>, vector<4xf32>, i32, i32, i32) -> vector<4xf32>
   amdgpu.mfma %arg3 * %arg3 + %arg2 { abid = 0 : i32, cbsz = 0 : i32, k = 32 : i32, m = 16 : i32, n = 16 : i32, blocks = 1 : i32 }  blgp = none : vector<8xbf16>, vector<8xbf16>, vector<4xf32>
   // CHECK: rocdl.mfma.i32.32x32x32.i8{{.*}}: (vector<4xi32>, vector<4xi32>, vector<16xi32>, i32, i32, i32) -> vector<16xi32>
   amdgpu.mfma %arg4 * %arg4 + %arg5 { abid = 0 : i32, cbsz = 0 : i32, k = 32 : i32, m = 32 : i32, n = 32 : i32, blocks = 1 : i32 }  blgp = none : vector<16xi8>, vector<16xi8>, vector<16xi32>
-- 
cgit v1.2.3


From 74ec1c287a88dffc232c38e0fdd3251f6b167d15 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Thu, 19 Jun 2025 18:56:23 +0100
Subject: [LV] Add tests interleaving extended and multiply/accumulate
 reductions.

Add missing test coverage for interleaving with
VPExtendedReduction/VPMulAccumulateReduction recipes.

Adds missing test coverage in preparation for
https://github.com/llvm/llvm-project/pull/144281.
---
 .../LoopVectorize/ARM/mve-reductions-interleave.ll | 153 +++++++++++++++++++++
 1 file changed, 153 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll

diff --git a/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll
new file mode 100644
index 000000000000..e27b0288b62e
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/ARM/mve-reductions-interleave.ll
@@ -0,0 +1,153 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize,instcombine,simplifycfg -simplifycfg-require-and-preserve-domtree=1 -tail-predication=enabled -force-vector-interleave=2 < %s -S -o - | FileCheck %s
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+target triple = "thumbv8.1m.main-arm-none-eabi"
+
+; 4x to use VADDLV
+define i64 @add_i32_i64(ptr nocapture readonly %x, i32 %n) #0 {
+; CHECK-LABEL: @add_i32_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -8
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i32 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP8]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <4 x i32> [[WIDE_LOAD]] to <4 x i64>
+; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP1]])
+; CHECK-NEXT:    [[TMP3]] = add i64 [[TMP2]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <4 x i32> [[WIDE_LOAD2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP9]])
+; CHECK-NEXT:    [[TMP7]] = add i64 [[TMP6]], [[VEC_PHI1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP4]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add i64 [[TMP7]], [[TMP3]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I_08:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_07:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_08]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[TMP5]] to i64
+; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_07]], [[CONV]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_08]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %red = phi i64 [ 0, %entry ], [ %red.next, %loop ]
+  %gep.x = getelementptr inbounds i32, ptr %x, i32 %iv
+  %0 = load i32, ptr %gep.x, align 4
+  %conv = sext i32 %0 to i64
+  %red.next = add nsw i64 %red, %conv
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  %red.lcssa = phi i64 [ %red.next, %loop ]
+  ret i64 %red.lcssa
+}
+
+; 4x to use VMLAL.u32
+define i64 @mla_i32_i64(ptr nocapture readonly %x, ptr nocapture readonly %y, i32 %n) #0 {
+; CHECK-LABEL: @mla_i32_i64(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[N:%.*]], 8
+; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_VEC:%.*]] = and i32 [[N]], -8
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP5:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[X:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP0]], i32 16
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP12]], align 4
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[Y:%.*]], i32 [[INDEX]]
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 16
+; CHECK-NEXT:    [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4
+; CHECK-NEXT:    [[WIDE_LOAD4:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD1]], [[WIDE_LOAD]]
+; CHECK-NEXT:    [[TMP14:%.*]] = mul nsw <4 x i32> [[WIDE_LOAD4]], [[WIDE_LOAD2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sext <4 x i32> [[TMP2]] to <4 x i64>
+; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP3]])
+; CHECK-NEXT:    [[TMP5]] = add i64 [[TMP4]], [[VEC_PHI]]
+; CHECK-NEXT:    [[TMP9:%.*]] = sext <4 x i32> [[TMP14]] to <4 x i64>
+; CHECK-NEXT:    [[TMP10:%.*]] = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> [[TMP9]])
+; CHECK-NEXT:    [[TMP11]] = add i64 [[TMP10]], [[VEC_PHI1]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[BIN_RDX:%.*]] = add i64 [[TMP11]], [[TMP5]]
+; CHECK-NEXT:    [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[CMP_N]], label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER:%.*]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i64 [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ]
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[I_010:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[R_09:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[ADD:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 [[I_010]]
+; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
+; CHECK-NEXT:    [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[Y]], i32 [[I_010]]
+; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4
+; CHECK-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], [[TMP7]]
+; CHECK-NEXT:    [[CONV:%.*]] = sext i32 [[MUL]] to i64
+; CHECK-NEXT:    [[ADD]] = add nsw i64 [[R_09]], [[CONV]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_010]], 1
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]]
+; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       exit:
+; CHECK-NEXT:    [[R_0_LCSSA:%.*]] = phi i64 [ [[ADD]], [[FOR_BODY]] ], [ [[BIN_RDX]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    ret i64 [[R_0_LCSSA]]
+;
+entry:
+  br label %loop
+
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %red = phi i64 [ 0, %entry ], [ %red.next, %loop ]
+  %gep.x = getelementptr inbounds i32, ptr %x, i32 %iv
+  %0 = load i32, ptr %gep.x, align 4
+  %gep.y = getelementptr inbounds i32, ptr %y, i32 %iv
+  %1 = load i32, ptr %gep.y, align 4
+  %mul = mul nsw i32 %1, %0
+  %conv = sext i32 %mul to i64
+  %red.next = add nsw i64 %red, %conv
+  %iv.next = add nuw nsw i32 %iv, 1
+  %ec = icmp eq i32 %iv.next, %n
+  br i1 %ec, label %exit, label %loop
+
+exit:
+  %red.lcssa = phi i64 [ %red.next, %loop ]
+  ret i64 %red.lcssa
+}
+
+attributes #0 = { "target-features"="+mve" }
-- 
cgit v1.2.3


From 3bee9ba0156ee130fa88379a5a89de0812936a3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Nicolai=20H=C3=A4hnle?= <nicolai.haehnle@amd.com>
Date: Thu, 19 Jun 2025 11:22:49 -0700
Subject: AMDGPU/GFX12: Fix s_barrier_signal_isfirst for single-wave workgroups
 (#143634)

Barrier instructions are no-ops in single-wave workgroups. This includes
s_barrier_signal_isfirst, which will leave SCC unmodified.

Model this correctly (via an implicit use of SCC) and ensure SCC==1
before the barrier instruction (if the wave is the only one of the
workgroup, then it is the first).

---------

Co-authored-by: Matt Arsenault <arsenm2@gmail.com>
---
 llvm/docs/AMDGPUUsage.rst                          |  4 +++
 .../Target/AMDGPU/AMDGPUInstructionSelector.cpp    |  3 ++
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp          |  8 +++++
 llvm/lib/Target/AMDGPU/SOPInstructions.td          |  3 ++
 llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir    | 10 ++++--
 .../AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll | 41 ++++++++++++++++++++++
 6 files changed, 66 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index c052b076c21c..ed3e4c8513e2 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1402,6 +1402,10 @@ The AMDGPU backend implements the following LLVM IR intrinsics.
                                                    performs subtraction only if the memory value is greater than or
                                                    equal to the data value.
 
+  llvm.amdgcn.s.barrier.signal.isfirst             Provides access to the s_barrier_signal_first instruction;
+                                                   additionally ensures that the result value is valid even when the
+                                                   intrinsic is used from a wave that is not running in a workgroup.
+
   llvm.amdgcn.s.getpc                              Provides access to the s_getpc_b64 instruction, but with the return value
                                                    sign-extended from the width of the underlying PC hardware register even on
                                                    processors where the s_getpc_b64 instruction returns a zero-extended value.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index 7e72f6ca478f..672520390c8b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -5918,6 +5918,9 @@ bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
   const DebugLoc &DL = I.getDebugLoc();
   Register CCReg = I.getOperand(0).getReg();
 
+  // Set SCC to true, in case the barrier instruction gets converted to a NOP.
+  BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_CMP_EQ_U32)).addImm(0).addImm(0);
+
   BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
       .addImm(I.getOperand(2).getImm());
 
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 586de433ea28..b1e77a282e41 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -5423,6 +5423,14 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
     MI.eraseFromParent();
     return BB;
   }
+  case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: {
+    // Set SCC to true, in case the barrier instruction gets converted to a NOP.
+    BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(),
+            TII->get(AMDGPU::S_CMP_EQ_U32))
+        .addImm(0)
+        .addImm(0);
+    return BB;
+  }
   case AMDGPU::GET_GROUPSTATICSIZE: {
     assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA ||
            getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL);
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index e0a36758534d..90e65a6950c0 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -472,6 +472,7 @@ def S_BARRIER_SIGNAL_M0 : SOP1_Pseudo <"s_barrier_signal m0", (outs), (ins),
 def S_BARRIER_SIGNAL_ISFIRST_M0 : SOP1_Pseudo <"s_barrier_signal_isfirst m0", (outs), (ins),
   "", []>{
   let Defs = [SCC];
+  let Uses = [M0, SCC];
   let SchedRW = [WriteBarrier];
   let isConvergent = 1;
 }
@@ -487,6 +488,8 @@ def S_BARRIER_SIGNAL_IMM : SOP1_Pseudo <"s_barrier_signal", (outs),
 def S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_Pseudo <"s_barrier_signal_isfirst", (outs),
   (ins SplitBarrier:$src0), "$src0", [(set SCC, (int_amdgcn_s_barrier_signal_isfirst timm:$src0))]>{
   let Defs = [SCC];
+  let Uses = [SCC];
+  let usesCustomInserter = 1;
   let SchedRW = [WriteBarrier];
   let isConvergent = 1;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
index e4b16a3fa004..f437dee253d0 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
+++ b/llvm/test/CodeGen/AMDGPU/insert-skips-gfx12.mir
@@ -374,7 +374,8 @@ body: |
   ; CHECK-NEXT:   successors: %bb.2(0x80000000)
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   V_NOP_e32 implicit $exec
-  ; CHECK-NEXT:   S_BARRIER_SIGNAL_ISFIRST_IMM -1, implicit-def $scc
+  ; CHECK-NEXT:   S_CMP_EQ_U32 0, 0, implicit-def $scc
+  ; CHECK-NEXT:   S_BARRIER_SIGNAL_ISFIRST_IMM -1, implicit-def $scc, implicit $scc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
   ; CHECK-NEXT:   S_ENDPGM 0
@@ -385,7 +386,8 @@ body: |
   bb.1:
     successors: %bb.2
     V_NOP_e32 implicit $exec
-    S_BARRIER_SIGNAL_ISFIRST_IMM -1, implicit-def $scc
+    S_CMP_EQ_U32 0, 0, implicit-def $scc
+    S_BARRIER_SIGNAL_ISFIRST_IMM -1, implicit-def $scc, implicit $scc
 
   bb.2:
     S_ENDPGM 0
@@ -437,6 +439,7 @@ body: |
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT:   V_NOP_e32 implicit $exec
   ; CHECK-NEXT:   $m0 = S_MOV_B32 -1
+  ; CHECK-NEXT:   S_CMP_EQ_U32 0, 0, implicit-def $scc
   ; CHECK-NEXT:   S_BARRIER_SIGNAL_ISFIRST_M0 implicit $m0, implicit-def $scc
   ; CHECK-NEXT: {{  $}}
   ; CHECK-NEXT: bb.2:
@@ -449,7 +452,8 @@ body: |
     successors: %bb.2
     V_NOP_e32 implicit $exec
     $m0 = S_MOV_B32 -1
-    S_BARRIER_SIGNAL_ISFIRST_M0 implicit $m0, implicit-def $scc
+    S_CMP_EQ_U32 0, 0, implicit-def $scc
+    S_BARRIER_SIGNAL_ISFIRST_M0 implicit $m0, implicit-def $scc, implicit $scc
 
   bb.2:
     S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
new file mode 100644
index 000000000000..651d204f65b6
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.signal.isfirst.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12-SDAG %s
+; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200  < %s | FileCheck -check-prefixes=GFX12-GISEL %s
+
+define i1 @func1() {
+; GFX12-SDAG-LABEL: func1:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_expcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_samplecnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_cmp_eq_u32 0, 0
+; GFX12-SDAG-NEXT:    s_wait_storecnt 0x0
+; GFX12-SDAG-NEXT:    s_barrier_signal_isfirst -1
+; GFX12-SDAG-NEXT:    s_cselect_b32 s0, -1, 0
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX12-SDAG-NEXT:    s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-GISEL-LABEL: func1:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_expcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_samplecnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_bvhcnt 0x0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_cmp_eq_u32 0, 0
+; GFX12-GISEL-NEXT:    s_wait_storecnt 0x0
+; GFX12-GISEL-NEXT:    s_barrier_signal_isfirst -1
+; GFX12-GISEL-NEXT:    s_cselect_b32 s0, 1, 0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT:    s_wait_kmcnt 0x0
+; GFX12-GISEL-NEXT:    s_setpc_b64 s[30:31]
+  %r = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1)
+  ret i1 %r
+}
+
+declare i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32)
-- 
cgit v1.2.3


From 633e740e3453bab06bf535830174c759100257f9 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Thu, 19 Jun 2025 14:32:20 -0400
Subject: [Clang][AMDGPU][Driver] Add `avail-extern-gv-in-addrspace-to-local`
 option when ThinTLO is enabled (#144914)

On AMDGPU, we need an extra argument
`-avail-extern-gv-in-addrspace-to-local=3`
to privatize LDS global variables when ThinLTO is enabled.
---
 clang/lib/Driver/ToolChains/Clang.cpp  | 3 +++
 clang/lib/Driver/ToolChains/HIPAMD.cpp | 2 ++
 clang/test/Driver/hip-thinlto.hip      | 1 +
 clang/test/Driver/openmp-offload-gpu.c | 1 +
 4 files changed, 7 insertions(+)

diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 5a3c09e3a343..e910a2bedeeb 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -9192,6 +9192,9 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
           CmdArgs.push_back(
               Args.MakeArgString("--device-linker=" + TC->getTripleString() +
                                  "=-plugin-opt=-avail-extern-to-local"));
+          CmdArgs.push_back(Args.MakeArgString(
+              "--device-linker=" + TC->getTripleString() +
+              "=-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3"));
           if (Kind == Action::OFK_OpenMP) {
             CmdArgs.push_back(
                 Args.MakeArgString("--device-linker=" + TC->getTripleString() +
diff --git a/clang/lib/Driver/ToolChains/HIPAMD.cpp b/clang/lib/Driver/ToolChains/HIPAMD.cpp
index 74ac8306e7cc..b8f3a70ee827 100644
--- a/clang/lib/Driver/ToolChains/HIPAMD.cpp
+++ b/clang/lib/Driver/ToolChains/HIPAMD.cpp
@@ -102,6 +102,8 @@ void AMDGCN::Linker::constructLldCommand(Compilation &C, const JobAction &JA,
   if (IsThinLTO) {
     LldArgs.push_back(Args.MakeArgString("-plugin-opt=-force-import-all"));
     LldArgs.push_back(Args.MakeArgString("-plugin-opt=-avail-extern-to-local"));
+    LldArgs.push_back(Args.MakeArgString(
+        "-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3"));
   }
 
   for (const Arg *A : Args.filtered(options::OPT_mllvm)) {
diff --git a/clang/test/Driver/hip-thinlto.hip b/clang/test/Driver/hip-thinlto.hip
index 4227cd3f2e9f..bcb7d4e6cb52 100644
--- a/clang/test/Driver/hip-thinlto.hip
+++ b/clang/test/Driver/hip-thinlto.hip
@@ -3,6 +3,7 @@
 // CHECK: -plugin-opt=thinlto
 // CHECK-SAME: -plugin-opt=-force-import-all
 // CHECK-SAME: -plugin-opt=-avail-extern-to-local
+// CHECK-SAME: -plugin-opt=-avail-extern-gv-in-addrspace-to-local=3
 int main(int, char *[]) {
   return 0;
 }
diff --git a/clang/test/Driver/openmp-offload-gpu.c b/clang/test/Driver/openmp-offload-gpu.c
index f67c2173cb14..2af3e2da3b21 100644
--- a/clang/test/Driver/openmp-offload-gpu.c
+++ b/clang/test/Driver/openmp-offload-gpu.c
@@ -388,6 +388,7 @@
 // THINLTO-GFX906: --device-compiler=amdgcn-amd-amdhsa=-flto=thin
 // THINLTO-GFX906-SAME: --device-linker=amdgcn-amd-amdhsa=-plugin-opt=-force-import-all
 // THINLTO-GFX906-SAME: --device-linker=amdgcn-amd-amdhsa=-plugin-opt=-avail-extern-to-local
+// THINLTO-GFX906-SAME: --device-linker=amdgcn-amd-amdhsa=-plugin-opt=-avail-extern-gv-in-addrspace-to-local=3
 // THINLTO-GFX906-SAME: --device-linker=amdgcn-amd-amdhsa=-plugin-opt=-amdgpu-internalize-symbols
 //
 // RUN:   %clang -### --target=x86_64-unknown-linux-gnu -fopenmp=libomp \
-- 
cgit v1.2.3


From baf35d7a829efb9688dc0aef6d1e161ef6bc5983 Mon Sep 17 00:00:00 2001
From: William Huynh <William.Huynh@arm.com>
Date: Thu, 19 Jun 2025 19:50:07 +0100
Subject: [libc] Fix bug in LIBC_CONF_ERRNO_MODE being undefined (#144896)

A typo, set() instead of list() would cause the build to not define
LIBC_CONF_ERRNO_MODE, which would cause the wrong configuration to be
used.
---
 libc/cmake/modules/LLVMLibCCompileOptionRules.cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
index a98e7276bef8..82d06e2b9eb5 100644
--- a/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
+++ b/libc/cmake/modules/LLVMLibCCompileOptionRules.cmake
@@ -107,7 +107,7 @@ function(_get_compile_options_from_config output_var)
   endif()
 
   if(LIBC_CONF_ERRNO_MODE)
-    set(APPEND config_options "-DLIBC_ERRNO_MODE=${LIBC_CONF_ERRNO_MODE}")
+    list(APPEND config_options "-DLIBC_ERRNO_MODE=${LIBC_CONF_ERRNO_MODE}")
   endif()
 
   set(${output_var} ${config_options} PARENT_SCOPE)
-- 
cgit v1.2.3


From 8631b4f1b4f30edd1f26b20e35b7367517aba359 Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 19 Jun 2025 12:12:04 -0700
Subject: [flang] Set low probability for array repacking code. (#144830)

This allows LLVM to place the most probably cold blocks
that do the repacking out of the line of the potentially hot code.
---
 flang/include/flang/Optimizer/Dialect/FIROps.td   | 10 ++++
 flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp | 34 +++++++-----
 flang/test/Integration/cold_array_repacking.f90   | 30 +++++++++++
 flang/test/Transforms/lower-repack-arrays.fir     | 64 +++++++++++------------
 4 files changed, 92 insertions(+), 46 deletions(-)
 create mode 100644 flang/test/Integration/cold_array_repacking.f90

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 27a6ca4ebdb4..8ac847dd7dd0 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2381,6 +2381,16 @@ def fir_IfOp
     static constexpr llvm::StringRef getWeightsAttrAssemblyName() {
       return "weights";
     }
+
+    /// Sets WeightedRegionBranchOpInterface weights to indicate
+    /// that either THEN or ELSE branch is unlikely.
+    /// By default, THEN branch is set to be unlikely.
+    void setUnlikelyIfWeights(bool unlikelyElse = false) {
+      if (unlikelyElse)
+        setWeights({1, 0});
+      else
+        setWeights({0, 1});
+    }
   }];
 }
 
diff --git a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
index de97a0bbc184..2774382c22bf 100644
--- a/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
+++ b/flang/lib/Optimizer/CodeGen/LowerRepackArrays.cpp
@@ -250,6 +250,8 @@ PackArrayConversion::genRepackedBox(fir::FirOpBuilder &builder,
 
   fir::IfOp ifOp =
       builder.create<fir::IfOp>(loc, boxType, doPack, /*withElseRegion=*/true);
+  // Assume that the repacking is unlikely.
+  ifOp.setUnlikelyIfWeights();
 
   // Return original box.
   builder.setInsertionPointToStart(&ifOp.getElseRegion().front());
@@ -322,20 +324,24 @@ UnpackArrayConversion::matchAndRewrite(fir::UnpackArrayOp op,
 
     auto isNotSame = builder.genPtrCompare(loc, mlir::arith::CmpIPredicate::ne,
                                            tempAddr, originalAddr);
-    builder.genIfThen(loc, isNotSame).genThen([&]() {});
-    // Copy from temporary to the original.
-    if (!op.getNoCopy())
-      fir::runtime::genShallowCopy(builder, loc, originalBox, tempBox,
-                                   /*resultIsAllocated=*/true);
-
-    // Deallocate, if it was allocated in heap.
-    // Note that the stack attribute does not always mean
-    // that the allocation was actually done in stack memory.
-    // There are currently cases where we delegate the allocation
-    // to the runtime that uses heap memory, even when the stack
-    // attribute is set on fir.pack_array.
-    if (!op.getStack() || !canAllocateTempOnStack(originalBox))
-      builder.create<fir::FreeMemOp>(loc, tempAddr);
+    builder.genIfThen(loc, isNotSame)
+        .genThen([&]() {
+          // Copy from temporary to the original.
+          if (!op.getNoCopy())
+            fir::runtime::genShallowCopy(builder, loc, originalBox, tempBox,
+                                         /*resultIsAllocated=*/true);
+
+          // Deallocate, if it was allocated in heap.
+          // Note that the stack attribute does not always mean
+          // that the allocation was actually done in stack memory.
+          // There are currently cases where we delegate the allocation
+          // to the runtime that uses heap memory, even when the stack
+          // attribute is set on fir.pack_array.
+          if (!op.getStack() || !canAllocateTempOnStack(originalBox))
+            builder.create<fir::FreeMemOp>(loc, tempAddr);
+        })
+        .getIfOp()
+        .setUnlikelyIfWeights();
   });
   rewriter.eraseOp(op);
   return mlir::success();
diff --git a/flang/test/Integration/cold_array_repacking.f90 b/flang/test/Integration/cold_array_repacking.f90
new file mode 100644
index 000000000000..0f25fcae866e
--- /dev/null
+++ b/flang/test/Integration/cold_array_repacking.f90
@@ -0,0 +1,30 @@
+! Check that the branch weights used by the array repacking
+! are propagated all the way to LLVM IR:
+! RUN: %flang_fc1 -frepack-arrays -emit-llvm %s -o - | FileCheck %s
+
+! CHECK-LABEL: define void @test_(
+! CHECK-SAME:      ptr [[TMP0:%.*]]) {
+! CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP0]] to i64
+! CHECK:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
+! CHECK:    br i1 [[TMP5]], label %[[BB6:.*]], label %[[BB46:.*]]
+! CHECK:  [[BB6]]:
+! CHECK:    [[TMP7:%.*]] = call i1 @_FortranAIsContiguous(ptr [[TMP0]])
+! CHECK:    [[TMP8:%.*]] = icmp eq i1 [[TMP7]], false
+! CHECK:    [[TMP13:%.*]] = and i1 [[TMP8]], [[TMP12:.*]]
+! CHECK:    br i1 [[TMP13]], label %[[BB14:.*]], label %[[BB46]], !prof [[PROF2:![0-9]+]]
+! CHECK:  [[BB14]]:
+! CHECK:    call void @_FortranAShallowCopyDirect
+! CHECK:    br label %[[BB46]]
+! CHECK:  [[BB46]]:
+! CHECK:    br i1 [[TMP5]], label %[[BB48:.*]], label %[[BB57:.*]]
+! CHECK:  [[BB48]]:
+! CHECK:    br i1 [[TMP55:.*]], label %[[BB56:.*]], label %[[BB57]], !prof [[PROF2]]
+! CHECK:  [[BB56]]:
+! CHECK:    call void @_FortranAShallowCopyDirect
+! CHECK:    br label %[[BB57]]
+! CHECK:  [[BB57]]:
+! CHECK:    ret void
+! CHECK: [[PROF2]] = !{!"branch_weights", i32 0, i32 1}
+subroutine test(x)
+  real :: x(:)
+end subroutine test
diff --git a/flang/test/Transforms/lower-repack-arrays.fir b/flang/test/Transforms/lower-repack-arrays.fir
index 012e957173ac..458869cce45f 100644
--- a/flang/test/Transforms/lower-repack-arrays.fir
+++ b/flang/test/Transforms/lower-repack-arrays.fir
@@ -22,7 +22,7 @@ func.func @_QPtest1(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"})
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.box<!fir.array<?x?xf32>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?xf32>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?xf32>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?xf32>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -52,7 +52,7 @@ func.func @_QPtest1(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x"})
 // CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_32]] : index
-// CHECK:             fir.if %[[VAL_33]] {
+// CHECK:             fir.if %[[VAL_33]] weights([0, 1]) {
 // CHECK:               %[[VAL_34:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
@@ -87,7 +87,7 @@ func.func @_QPtest1_whole(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name =
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.box<!fir.array<?x?xf32>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?xf32>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?xf32>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?xf32>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -117,7 +117,7 @@ func.func @_QPtest1_whole(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name =
 // CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_32]] : index
-// CHECK:             fir.if %[[VAL_33]] {
+// CHECK:             fir.if %[[VAL_33]] weights([0, 1]) {
 // CHECK:               %[[VAL_34:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
@@ -150,7 +150,7 @@ func.func @_QPtest1_in(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x
 // CHECK:             %[[VAL_10:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.box<!fir.array<?x?xf32>>>
 // CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_10]] : (!fir.ref<!fir.box<!fir.array<?x?xf32>>>) -> i1
 // CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_11]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] -> (!fir.box<!fir.array<?x?xf32>>) {
+// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] weights([0, 1]) -> (!fir.box<!fir.array<?x?xf32>>) {
 // CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]] = fir.shape %[[VAL_14]]#1, %[[VAL_15]]#1 : (index, index) -> !fir.shape<2>
@@ -180,7 +180,7 @@ func.func @_QPtest1_in(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "x
 // CHECK:             %[[VAL_29:.*]] = fir.convert %[[VAL_28]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_32:.*]] = arith.cmpi ne, %[[VAL_29]], %[[VAL_31]] : index
-// CHECK:             fir.if %[[VAL_32]] {
+// CHECK:             fir.if %[[VAL_32]] weights([0, 1]) {
 // CHECK:               fir.freemem %[[VAL_28]] : !fir.heap<!fir.array<?x?xf32>>
 // CHECK:             }
 // CHECK:           }
@@ -209,7 +209,7 @@ func.func @_QPtest1_out(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "
 // CHECK:             %[[VAL_10:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.ref<!fir.box<!fir.array<?x?xf32>>>
 // CHECK:             %[[VAL_11:.*]] = fir.is_present %[[VAL_10]] : (!fir.ref<!fir.box<!fir.array<?x?xf32>>>) -> i1
 // CHECK:             %[[VAL_12:.*]] = arith.andi %[[VAL_9]], %[[VAL_11]] : i1
-// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] -> (!fir.box<!fir.array<?x?xf32>>) {
+// CHECK:             %[[VAL_13:.*]] = fir.if %[[VAL_12]] weights([0, 1]) -> (!fir.box<!fir.array<?x?xf32>>) {
 // CHECK:               %[[VAL_14:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_2]] : (!fir.box<!fir.array<?x?xf32>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]] = fir.shape %[[VAL_14]]#1, %[[VAL_15]]#1 : (index, index) -> !fir.shape<2>
@@ -234,7 +234,7 @@ func.func @_QPtest1_out(%arg0: !fir.box<!fir.array<?x?xf32>> {fir.bindc_name = "
 // CHECK:             %[[VAL_25:.*]] = fir.convert %[[VAL_24]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_27:.*]] = fir.convert %[[VAL_26]] : (!fir.heap<!fir.array<?x?xf32>>) -> index
 // CHECK:             %[[VAL_28:.*]] = arith.cmpi ne, %[[VAL_25]], %[[VAL_27]] : index
-// CHECK:             fir.if %[[VAL_28]] {
+// CHECK:             fir.if %[[VAL_28]] weights([0, 1]) {
 // CHECK:               %[[VAL_29:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_30:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
 // CHECK:               %[[VAL_31:.*]] = fir.convert %[[VAL_7]] : (!fir.box<!fir.array<?x?xf32>>) -> !fir.box<none>
@@ -280,7 +280,7 @@ func.func @_QPtest2(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.box
 // CHECK:             %[[VAL_17:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>
 // CHECK:             %[[VAL_18:.*]] = fir.is_present %[[VAL_17]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>) -> i1
 // CHECK:             %[[VAL_19:.*]] = arith.andi %[[VAL_16]], %[[VAL_18]] : i1
-// CHECK:             %[[VAL_20:.*]] = fir.if %[[VAL_19]] -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
+// CHECK:             %[[VAL_20:.*]] = fir.if %[[VAL_19]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
 // CHECK:               %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_22:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_23:.*]] = fir.shape %[[VAL_21]]#1, %[[VAL_22]]#1 : (index, index) -> !fir.shape<2>
@@ -310,7 +310,7 @@ func.func @_QPtest2(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !fir.box
 // CHECK:             %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_39:.*]] = arith.cmpi ne, %[[VAL_36]], %[[VAL_38]] : index
-// CHECK:             fir.if %[[VAL_39]] {
+// CHECK:             fir.if %[[VAL_39]] weights([0, 1]) {
 // CHECK:               %[[VAL_40:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_14]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
@@ -356,7 +356,7 @@ func.func @_QPtest2_stack(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !f
 // CHECK:             %[[VAL_17:.*]] = fir.box_addr %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>
 // CHECK:             %[[VAL_18:.*]] = fir.is_present %[[VAL_17]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>) -> i1
 // CHECK:             %[[VAL_19:.*]] = arith.andi %[[VAL_16]], %[[VAL_18]] : i1
-// CHECK:             %[[VAL_20:.*]] = fir.if %[[VAL_19]] -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
+// CHECK:             %[[VAL_20:.*]] = fir.if %[[VAL_19]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
 // CHECK:               %[[VAL_21:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_5]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_22:.*]]:3 = fir.box_dims %[[VAL_1]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_23:.*]] = fir.shape %[[VAL_21]]#1, %[[VAL_22]]#1 : (index, index) -> !fir.shape<2>
@@ -386,7 +386,7 @@ func.func @_QPtest2_stack(%arg0: !fir.ref<i32> {fir.bindc_name = "n"}, %arg1: !f
 // CHECK:             %[[VAL_36:.*]] = fir.convert %[[VAL_35]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_38:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_39:.*]] = arith.cmpi ne, %[[VAL_36]], %[[VAL_38]] : index
-// CHECK:             fir.if %[[VAL_39]] {
+// CHECK:             fir.if %[[VAL_39]] weights([0, 1]) {
 // CHECK:               %[[VAL_40:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_41:.*]] = fir.convert %[[VAL_1]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_42:.*]] = fir.convert %[[VAL_14]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
@@ -420,7 +420,7 @@ func.func @_QPtest3(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.bindc_n
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.box_elesize %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> index
@@ -451,7 +451,7 @@ func.func @_QPtest3(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.bindc_n
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_34:.*]] = arith.cmpi ne, %[[VAL_31]], %[[VAL_33]] : index
-// CHECK:             fir.if %[[VAL_34]] {
+// CHECK:             fir.if %[[VAL_34]] weights([0, 1]) {
 // CHECK:               %[[VAL_35:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
@@ -486,7 +486,7 @@ func.func @_QPtest3_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.b
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,?>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.box_elesize %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> index
@@ -517,7 +517,7 @@ func.func @_QPtest3_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,?>>> {fir.b
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (!fir.heap<!fir.array<?x?x!fir.char<1,?>>>) -> index
 // CHECK:             %[[VAL_34:.*]] = arith.cmpi ne, %[[VAL_31]], %[[VAL_33]] : index
-// CHECK:             fir.if %[[VAL_34]] {
+// CHECK:             fir.if %[[VAL_34]] weights([0, 1]) {
 // CHECK:               %[[VAL_35:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?x!fir.char<1,?>>>) -> !fir.box<none>
@@ -553,7 +553,7 @@ func.func @_QPtest4(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.bindc_
 // CHECK:             %[[VAL_12:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,10>>>>
 // CHECK:             %[[VAL_13:.*]] = fir.is_present %[[VAL_12]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,10>>>>) -> i1
 // CHECK:             %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_13]] : i1
-// CHECK:             %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) {
+// CHECK:             %[[VAL_15:.*]] = fir.if %[[VAL_14]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) {
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_16]]#1, %[[VAL_17]]#1 : (index, index) -> !fir.shape<2>
@@ -583,7 +583,7 @@ func.func @_QPtest4(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.bindc_
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>) -> index
 // CHECK:             %[[VAL_34:.*]] = arith.cmpi ne, %[[VAL_31]], %[[VAL_33]] : index
-// CHECK:             fir.if %[[VAL_34]] {
+// CHECK:             fir.if %[[VAL_34]] weights([0, 1]) {
 // CHECK:               %[[VAL_35:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_9]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
@@ -620,7 +620,7 @@ func.func @_QPtest4_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.
 // CHECK:             %[[VAL_12:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,10>>>>
 // CHECK:             %[[VAL_13:.*]] = fir.is_present %[[VAL_12]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.char<1,10>>>>) -> i1
 // CHECK:             %[[VAL_14:.*]] = arith.andi %[[VAL_11]], %[[VAL_13]] : i1
-// CHECK:             %[[VAL_15:.*]] = fir.if %[[VAL_14]] -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) {
+// CHECK:             %[[VAL_15:.*]] = fir.if %[[VAL_14]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) {
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_18:.*]] = fir.shape %[[VAL_16]]#1, %[[VAL_17]]#1 : (index, index) -> !fir.shape<2>
@@ -650,7 +650,7 @@ func.func @_QPtest4_stack(%arg0: !fir.box<!fir.array<?x?x!fir.char<1,10>>> {fir.
 // CHECK:             %[[VAL_31:.*]] = fir.convert %[[VAL_30]] : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = fir.convert %[[VAL_32]] : (!fir.heap<!fir.array<?x?x!fir.char<1,10>>>) -> index
 // CHECK:             %[[VAL_34:.*]] = arith.cmpi ne, %[[VAL_31]], %[[VAL_33]] : index
-// CHECK:             fir.if %[[VAL_34]] {
+// CHECK:             fir.if %[[VAL_34]] weights([0, 1]) {
 // CHECK:               %[[VAL_35:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_37:.*]] = fir.convert %[[VAL_9]] : (!fir.box<!fir.array<?x?x!fir.char<1,10>>>) -> !fir.box<none>
@@ -684,7 +684,7 @@ func.func @_QPtest5(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bind
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -714,7 +714,7 @@ func.func @_QPtest5(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bind
 // CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_32]] : index
-// CHECK:             fir.if %[[VAL_33]] {
+// CHECK:             fir.if %[[VAL_33]] weights([0, 1]) {
 // CHECK:               %[[VAL_34:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
@@ -749,7 +749,7 @@ func.func @_QPtest5_stack(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fi
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_4]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[VAL_0]], %[[VAL_3]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -779,7 +779,7 @@ func.func @_QPtest5_stack(%arg0: !fir.box<!fir.array<?x?x!fir.type<_QMmTt>>> {fi
 // CHECK:             %[[VAL_30:.*]] = fir.convert %[[VAL_29]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_32:.*]] = fir.convert %[[VAL_31]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_33:.*]] = arith.cmpi ne, %[[VAL_30]], %[[VAL_32]] : index
-// CHECK:             fir.if %[[VAL_33]] {
+// CHECK:             fir.if %[[VAL_33]] weights([0, 1]) {
 // CHECK:               %[[VAL_34:.*]] = fir.address_of(@{{_QQcl.*}}
 // CHECK:               %[[VAL_35:.*]] = fir.convert %[[VAL_0]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_36:.*]] = fir.convert %[[VAL_8]] : (!fir.box<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
@@ -814,7 +814,7 @@ func.func @_QPtest6(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bi
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -853,7 +853,7 @@ func.func @_QPtest6(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {fir.bi
 // CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
-// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:             fir.if %[[VAL_41]] weights([0, 1]) {
 // CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 // CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
@@ -890,7 +890,7 @@ func.func @_QPtest6_stack(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -929,7 +929,7 @@ func.func @_QPtest6_stack(%arg0: !fir.class<!fir.array<?x?x!fir.type<_QMmTt>>> {
 // CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?x!fir.type<_QMmTt>>>) -> index
 // CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
-// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:             fir.if %[[VAL_41]] weights([0, 1]) {
 // CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 // CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
 // CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?x!fir.type<_QMmTt>>>) -> !fir.box<none>
@@ -965,7 +965,7 @@ func.func @_QPtest7(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.class<!fir.array<?x?xnone>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -1004,7 +1004,7 @@ func.func @_QPtest7(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_name = "x
 // CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
 // CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
 // CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
-// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:             fir.if %[[VAL_41]] weights([0, 1]) {
 // CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 // CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
@@ -1041,7 +1041,7 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:             %[[VAL_11:.*]] = fir.box_addr %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.ref<!fir.class<!fir.array<?x?xnone>>>
 // CHECK:             %[[VAL_12:.*]] = fir.is_present %[[VAL_11]] : (!fir.ref<!fir.class<!fir.array<?x?xnone>>>) -> i1
 // CHECK:             %[[VAL_13:.*]] = arith.andi %[[VAL_10]], %[[VAL_12]] : i1
-// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] -> (!fir.class<!fir.array<?x?xnone>>) {
+// CHECK:             %[[VAL_14:.*]] = fir.if %[[VAL_13]] weights([0, 1]) -> (!fir.class<!fir.array<?x?xnone>>) {
 // CHECK:               %[[VAL_15:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_3]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_16:.*]]:3 = fir.box_dims %[[ARG0]], %[[VAL_2]] : (!fir.class<!fir.array<?x?xnone>>, index) -> (index, index, index)
 // CHECK:               %[[VAL_17:.*]] = fir.shape %[[VAL_15]]#1, %[[VAL_16]]#1 : (index, index) -> !fir.shape<2>
@@ -1080,7 +1080,7 @@ func.func @_QPtest7_stack(%arg0: !fir.class<!fir.array<?x?xnone>> {fir.bindc_nam
 // CHECK:             %[[VAL_39:.*]] = fir.convert %[[VAL_37]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
 // CHECK:             %[[VAL_40:.*]] = fir.convert %[[VAL_38]] : (!fir.heap<!fir.array<?x?xnone>>) -> index
 // CHECK:             %[[VAL_41:.*]] = arith.cmpi ne, %[[VAL_39]], %[[VAL_40]] : index
-// CHECK:             fir.if %[[VAL_41]] {
+// CHECK:             fir.if %[[VAL_41]] weights([0, 1]) {
 // CHECK:               %[[VAL_42:.*]] = fir.address_of(@{{_QQcl.*}}) : !fir.ref<!fir.char<1,{{.*}}>>
 // CHECK:               %[[VAL_43:.*]] = fir.convert %[[ARG0]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
 // CHECK:               %[[VAL_44:.*]] = fir.convert %[[VAL_8]] : (!fir.class<!fir.array<?x?xnone>>) -> !fir.box<none>
-- 
cgit v1.2.3


From 28808dda2c53a1dff1076cb83a9b91d0866ebf9a Mon Sep 17 00:00:00 2001
From: Slava Zakharin <szakharin@nvidia.com>
Date: Thu, 19 Jun 2025 13:02:30 -0700
Subject: [flang] Fixed test added in #144830.

---
 flang/test/Integration/cold_array_repacking.f90 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flang/test/Integration/cold_array_repacking.f90 b/flang/test/Integration/cold_array_repacking.f90
index 0f25fcae866e..11b7d8c21b67 100644
--- a/flang/test/Integration/cold_array_repacking.f90
+++ b/flang/test/Integration/cold_array_repacking.f90
@@ -3,7 +3,7 @@
 ! RUN: %flang_fc1 -frepack-arrays -emit-llvm %s -o - | FileCheck %s
 
 ! CHECK-LABEL: define void @test_(
-! CHECK-SAME:      ptr [[TMP0:%.*]]) {
+! CHECK-SAME:      ptr [[TMP0:%.*]])
 ! CHECK:    [[TMP4:%.*]] = ptrtoint ptr [[TMP0]] to i64
 ! CHECK:    [[TMP5:%.*]] = icmp ne i64 [[TMP4]], 0
 ! CHECK:    br i1 [[TMP5]], label %[[BB6:.*]], label %[[BB46:.*]]
-- 
cgit v1.2.3


From dc058a3d84ed1bc4006416023e8b336f3214bdc7 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 13:17:23 -0700
Subject: [TableGen] Use ListSeparator (NFC) (#144936)

Note that an instance of ListSeparator evaluates to the empty string
for the first time and then ", " for subsequent references.
---
 clang/utils/TableGen/ClangAttrEmitter.cpp        | 6 ++----
 clang/utils/TableGen/ClangDiagnosticsEmitter.cpp | 9 +++------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp
index f892626a447e..dfeb6b1b1ec1 100644
--- a/clang/utils/TableGen/ClangAttrEmitter.cpp
+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp
@@ -5482,14 +5482,12 @@ void EmitTestPragmaAttributeSupportedAttributes(const RecordKeeper &Records,
     }
     const Record *SubjectObj = I.second->getValueAsDef("Subjects");
     OS << " (";
-    bool PrintComma = false;
+    ListSeparator LS;
     for (const auto &Subject :
          enumerate(SubjectObj->getValueAsListOfDefs("Subjects"))) {
       if (!isSupportedPragmaClangAttributeSubject(*Subject.value()))
         continue;
-      if (PrintComma)
-        OS << ", ";
-      PrintComma = true;
+      OS << LS;
       PragmaClangAttributeSupport::RuleOrAggregateRuleSet &RuleSet =
           Support.SubjectsToRules.find(Subject.value())->getSecond();
       if (RuleSet.isRule()) {
diff --git a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
index bfc60f485cd3..b28cb2c09ac5 100644
--- a/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
+++ b/clang/utils/TableGen/ClangDiagnosticsEmitter.cpp
@@ -2225,13 +2225,10 @@ void clang::EmitClangDiagDocs(const RecordKeeper &Records, raw_ostream &OS) {
       else
         OS << "Also controls ";
 
-      bool First = true;
       sort(GroupInfo.SubGroups);
-      for (StringRef Name : GroupInfo.SubGroups) {
-        if (!First) OS << ", ";
-        OS << "`" << (IsRemarkGroup ? "-R" : "-W") << Name << "`_";
-        First = false;
-      }
+      ListSeparator LS;
+      for (StringRef Name : GroupInfo.SubGroups)
+        OS << LS << "`" << (IsRemarkGroup ? "-R" : "-W") << Name << "`_";
       OS << ".\n\n";
     }
 
-- 
cgit v1.2.3


From 7349864d2c7c874c17ed546791489a46e896f901 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 13:17:31 -0700
Subject: [ADT] Remove an extraneous variable (NFC) (#144937)

Without this patch, Int and IntWord have the same value and type.
This patch removes the extraneous copy.
---
 llvm/include/llvm/ADT/PointerIntPair.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/include/llvm/ADT/PointerIntPair.h b/llvm/include/llvm/ADT/PointerIntPair.h
index f73f5bcd6ce0..9cfc65846d5b 100644
--- a/llvm/include/llvm/ADT/PointerIntPair.h
+++ b/llvm/include/llvm/ADT/PointerIntPair.h
@@ -206,11 +206,10 @@ struct PointerIntPairInfo {
   }
 
   static intptr_t updateInt(intptr_t OrigValue, intptr_t Int) {
-    intptr_t IntWord = static_cast<intptr_t>(Int);
-    assert((IntWord & ~IntMask) == 0 && "Integer too large for field");
+    assert((Int & ~IntMask) == 0 && "Integer too large for field");
 
     // Preserve all bits other than the ones we are updating.
-    return (OrigValue & ~ShiftedIntMask) | IntWord << IntShift;
+    return (OrigValue & ~ShiftedIntMask) | Int << IntShift;
   }
 };
 
-- 
cgit v1.2.3


From 3b672e1d7b7375ca2a048cfb252d0e8ff35724e2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 13:17:39 -0700
Subject: [llvm] Use "= delete" to delete constructors (NFC) (#144938)

None of the constructors touched in this patch has a corresponding
definition.  This patch explicitly deletes them with "= delete" while
moving them to the public section of respective classes.  Note that "=
delete" itself serves as documentation.

Identified with modernize-use-equals-delete.
---
 llvm/unittests/ADT/TestGraph.h     | 10 +++++-----
 llvm/unittests/Support/Casting.cpp |  4 +---
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/llvm/unittests/ADT/TestGraph.h b/llvm/unittests/ADT/TestGraph.h
index 3e6d4e14d5c1..a59ab504f714 100644
--- a/llvm/unittests/ADT/TestGraph.h
+++ b/llvm/unittests/ADT/TestGraph.h
@@ -24,14 +24,13 @@ namespace llvm {
 template <unsigned N>
 class Graph {
 private:
-  // Disable copying.
-  Graph(const Graph&);
-  Graph& operator=(const Graph&);
-
   static void ValidateIndex(unsigned Idx) {
     assert(Idx < N && "Invalid node index!");
   }
 public:
+  // Disable copying.
+  Graph(const Graph &) = delete;
+  Graph &operator=(const Graph &) = delete;
 
   /// NodeSubset - A subset of the graph's nodes.
   class NodeSubset {
@@ -169,11 +168,12 @@ public:
     /// yet been visited.
     NodeSubset Children;
 
-    ChildIterator(); // Disable default constructor.
   protected:
     ChildIterator(NodeType *F, NodeSubset C) : FirstNode(F), Children(C) {}
 
   public:
+    ChildIterator() = delete; // Disable default constructor.
+
     /// ChildIterator - Copy constructor.
     ChildIterator(const ChildIterator &other) = default;
     ChildIterator &operator=(const ChildIterator &other) = default;
diff --git a/llvm/unittests/Support/Casting.cpp b/llvm/unittests/Support/Casting.cpp
index a128cedaf398..18327f6dd167 100644
--- a/llvm/unittests/Support/Casting.cpp
+++ b/llvm/unittests/Support/Casting.cpp
@@ -24,13 +24,11 @@ template <typename T> IllegalCast *cast(...) { return nullptr; }
 //
 struct bar {
   bar() {}
+  bar(const bar &) = delete;
   struct foo *baz();
   struct foo *caz();
   struct foo *daz();
   struct foo *naz();
-
-private:
-  bar(const bar &);
 };
 struct foo {
   foo(const bar &) {}
-- 
cgit v1.2.3


From 03692aa40487d0f4090d329ca2904fb888242c94 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 13:17:47 -0700
Subject: [memprof] Use a lambda instead of std::bind (NFC) (#144940)

A lambda is a lot shorter than std::bind here.
---
 llvm/include/llvm/ProfileData/MemProfReader.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/llvm/include/llvm/ProfileData/MemProfReader.h b/llvm/include/llvm/ProfileData/MemProfReader.h
index 4d41d05b1457..25578ecd06f1 100644
--- a/llvm/include/llvm/ProfileData/MemProfReader.h
+++ b/llvm/include/llvm/ProfileData/MemProfReader.h
@@ -62,8 +62,7 @@ public:
       return make_error<InstrProfError>(instrprof_error::eof);
 
     if (Callback == nullptr)
-      Callback =
-          std::bind(&MemProfReader::idToFrame, this, std::placeholders::_1);
+      Callback = [&](FrameId Id) { return idToFrame(Id); };
 
     CallStackIdConverter<decltype(MemProfData.CallStacks)> CSIdConv(
         MemProfData.CallStacks, Callback);
-- 
cgit v1.2.3


From 9fd22cb56d4c626769afd938e0f9ef6157164394 Mon Sep 17 00:00:00 2001
From: Peter Klausler <pklausler@nvidia.com>
Date: Thu, 19 Jun 2025 13:42:46 -0700
Subject: [flang][NFC] Move new code to right place (#144551)

Some new code was added to flang/Semantics that only depends on
facilities in flang/Evaluate. Move it into Evaluate and clean up some
minor stylistic problems.
---
 flang/include/flang/Evaluate/tools.h        | 148 +++++++++++++
 flang/include/flang/Semantics/tools.h       | 149 -------------
 flang/lib/Evaluate/tools.cpp                | 311 ++++++++++++++++++++++++++
 flang/lib/Lower/OpenACC.cpp                 |   2 +-
 flang/lib/Lower/OpenMP/OpenMP.cpp           |   7 +-
 flang/lib/Semantics/check-omp-structure.cpp |   3 +
 flang/lib/Semantics/tools.cpp               | 329 ----------------------------
 7 files changed, 467 insertions(+), 482 deletions(-)

diff --git a/flang/include/flang/Evaluate/tools.h b/flang/include/flang/Evaluate/tools.h
index 1959d5f3a589..e04621f71f9a 100644
--- a/flang/include/flang/Evaluate/tools.h
+++ b/flang/include/flang/Evaluate/tools.h
@@ -1389,6 +1389,154 @@ inline bool HasCUDAImplicitTransfer(const Expr<SomeType> &expr) {
   return (hasConstant || (hostSymbols > 0)) && deviceSymbols > 0;
 }
 
+// Checks whether the symbol on the LHS is present in the RHS expression.
+bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs);
+
+namespace operation {
+
+enum class Operator {
+  Unknown,
+  Add,
+  And,
+  Associated,
+  Call,
+  Constant,
+  Convert,
+  Div,
+  Eq,
+  Eqv,
+  False,
+  Ge,
+  Gt,
+  Identity,
+  Intrinsic,
+  Le,
+  Lt,
+  Max,
+  Min,
+  Mul,
+  Ne,
+  Neqv,
+  Not,
+  Or,
+  Pow,
+  Resize, // Convert within the same TypeCategory
+  Sub,
+  True,
+};
+
+std::string ToString(Operator op);
+
+template <typename... Ts, int Kind>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::LogicalOperation<Kind>, Ts...> &op) {
+  switch (op.derived().logicalOperator) {
+  case common::LogicalOperator::And:
+    return Operator::And;
+  case common::LogicalOperator::Or:
+    return Operator::Or;
+  case common::LogicalOperator::Eqv:
+    return Operator::Eqv;
+  case common::LogicalOperator::Neqv:
+    return Operator::Neqv;
+  case common::LogicalOperator::Not:
+    return Operator::Not;
+  }
+  return Operator::Unknown;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Relational<T>, Ts...> &op) {
+  switch (op.derived().opr) {
+  case common::RelationalOperator::LT:
+    return Operator::Lt;
+  case common::RelationalOperator::LE:
+    return Operator::Le;
+  case common::RelationalOperator::EQ:
+    return Operator::Eq;
+  case common::RelationalOperator::NE:
+    return Operator::Ne;
+  case common::RelationalOperator::GE:
+    return Operator::Ge;
+  case common::RelationalOperator::GT:
+    return Operator::Gt;
+  }
+  return Operator::Unknown;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(const evaluate::Operation<evaluate::Add<T>, Ts...> &op) {
+  return Operator::Add;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Subtract<T>, Ts...> &op) {
+  return Operator::Sub;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Multiply<T>, Ts...> &op) {
+  return Operator::Mul;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Divide<T>, Ts...> &op) {
+  return Operator::Div;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Power<T>, Ts...> &op) {
+  return Operator::Pow;
+}
+
+template <typename T, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::RealToIntPower<T>, Ts...> &op) {
+  return Operator::Pow;
+}
+
+template <typename T, common::TypeCategory C, typename... Ts>
+Operator OperationCode(
+    const evaluate::Operation<evaluate::Convert<T, C>, Ts...> &op) {
+  if constexpr (C == T::category) {
+    return Operator::Resize;
+  } else {
+    return Operator::Convert;
+  }
+}
+
+template <typename T> Operator OperationCode(const evaluate::Constant<T> &x) {
+  return Operator::Constant;
+}
+
+template <typename T> Operator OperationCode(const T &) {
+  return Operator::Unknown;
+}
+
+Operator OperationCode(const evaluate::ProcedureDesignator &proc);
+
+} // namespace operation
+
+// Return information about the top-level operation (ignoring parentheses):
+// the operation code and the list of arguments.
+std::pair<operation::Operator, std::vector<Expr<SomeType>>>
+GetTopLevelOperation(const Expr<SomeType> &expr);
+
+// Check if expr is same as x, or a sequence of Convert operations on x.
+bool IsSameOrConvertOf(const Expr<SomeType> &expr, const Expr<SomeType> &x);
+
+// Strip away any top-level Convert operations (if any exist) and return
+// the input value. A ComplexConstructor(x, 0) is also considered as a
+// convert operation.
+// If the input is not Operation, Designator, FunctionRef or Constant,
+// it returns std::nullopt.
+std::optional<Expr<SomeType>> GetConvertInput(const Expr<SomeType> &x);
+
 } // namespace Fortran::evaluate
 
 namespace Fortran::semantics {
diff --git a/flang/include/flang/Semantics/tools.h b/flang/include/flang/Semantics/tools.h
index 69375a83dec2..f3cfa9b99fb4 100644
--- a/flang/include/flang/Semantics/tools.h
+++ b/flang/include/flang/Semantics/tools.h
@@ -756,154 +756,5 @@ std::string GetCommonBlockObjectName(const Symbol &, bool underscoring);
 // Check for ambiguous USE associations
 bool HadUseError(SemanticsContext &, SourceName at, const Symbol *);
 
-// Checks whether the symbol on the LHS is present in the RHS expression.
-bool CheckForSymbolMatch(const SomeExpr *lhs, const SomeExpr *rhs);
-
-namespace operation {
-
-enum class Operator {
-  Unknown,
-  Add,
-  And,
-  Associated,
-  Call,
-  Constant,
-  Convert,
-  Div,
-  Eq,
-  Eqv,
-  False,
-  Ge,
-  Gt,
-  Identity,
-  Intrinsic,
-  Le,
-  Lt,
-  Max,
-  Min,
-  Mul,
-  Ne,
-  Neqv,
-  Not,
-  Or,
-  Pow,
-  Resize, // Convert within the same TypeCategory
-  Sub,
-  True,
-};
-
-std::string ToString(Operator op);
-
-template <typename... Ts, int Kind>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::LogicalOperation<Kind>, Ts...> &op) {
-  switch (op.derived().logicalOperator) {
-  case common::LogicalOperator::And:
-    return Operator::And;
-  case common::LogicalOperator::Or:
-    return Operator::Or;
-  case common::LogicalOperator::Eqv:
-    return Operator::Eqv;
-  case common::LogicalOperator::Neqv:
-    return Operator::Neqv;
-  case common::LogicalOperator::Not:
-    return Operator::Not;
-  }
-  return Operator::Unknown;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Relational<T>, Ts...> &op) {
-  switch (op.derived().opr) {
-  case common::RelationalOperator::LT:
-    return Operator::Lt;
-  case common::RelationalOperator::LE:
-    return Operator::Le;
-  case common::RelationalOperator::EQ:
-    return Operator::Eq;
-  case common::RelationalOperator::NE:
-    return Operator::Ne;
-  case common::RelationalOperator::GE:
-    return Operator::Ge;
-  case common::RelationalOperator::GT:
-    return Operator::Gt;
-  }
-  return Operator::Unknown;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(const evaluate::Operation<evaluate::Add<T>, Ts...> &op) {
-  return Operator::Add;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Subtract<T>, Ts...> &op) {
-  return Operator::Sub;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Multiply<T>, Ts...> &op) {
-  return Operator::Mul;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Divide<T>, Ts...> &op) {
-  return Operator::Div;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Power<T>, Ts...> &op) {
-  return Operator::Pow;
-}
-
-template <typename T, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::RealToIntPower<T>, Ts...> &op) {
-  return Operator::Pow;
-}
-
-template <typename T, common::TypeCategory C, typename... Ts>
-Operator OperationCode(
-    const evaluate::Operation<evaluate::Convert<T, C>, Ts...> &op) {
-  if constexpr (C == T::category) {
-    return Operator::Resize;
-  } else {
-    return Operator::Convert;
-  }
-}
-
-template <typename T> //
-Operator OperationCode(const evaluate::Constant<T> &x) {
-  return Operator::Constant;
-}
-
-template <typename T> //
-Operator OperationCode(const T &) {
-  return Operator::Unknown;
-}
-
-Operator OperationCode(const evaluate::ProcedureDesignator &proc);
-
-} // namespace operation
-
-/// Return information about the top-level operation (ignoring parentheses):
-/// the operation code and the list of arguments.
-std::pair<operation::Operator, std::vector<SomeExpr>> GetTopLevelOperation(
-    const SomeExpr &expr);
-
-/// Check if expr is same as x, or a sequence of Convert operations on x.
-bool IsSameOrConvertOf(const SomeExpr &expr, const SomeExpr &x);
-
-/// Strip away any top-level Convert operations (if any exist) and return
-/// the input value. A ComplexConstructor(x, 0) is also considered as a
-/// convert operation.
-/// If the input is not Operation, Designator, FunctionRef or Constant,
-/// it returns std::nullopt.
-MaybeExpr GetConvertInput(const SomeExpr &x);
 } // namespace Fortran::semantics
 #endif // FORTRAN_SEMANTICS_TOOLS_H_
diff --git a/flang/lib/Evaluate/tools.cpp b/flang/lib/Evaluate/tools.cpp
index 222c32a9c332..68838564f87b 100644
--- a/flang/lib/Evaluate/tools.cpp
+++ b/flang/lib/Evaluate/tools.cpp
@@ -13,6 +13,7 @@
 #include "flang/Evaluate/traverse.h"
 #include "flang/Parser/message.h"
 #include "flang/Semantics/tools.h"
+#include "llvm/ADT/StringSwitch.h"
 #include <algorithm>
 #include <variant>
 
@@ -1595,6 +1596,316 @@ bool CheckForCoindexedObject(parser::ContextualMessages &messages,
   }
 }
 
+bool CheckForSymbolMatch(const Expr<SomeType> *lhs, const Expr<SomeType> *rhs) {
+  if (lhs && rhs) {
+    if (SymbolVector lhsSymbols{GetSymbolVector(*lhs)}; !lhsSymbols.empty()) {
+      const Symbol &first{*lhsSymbols.front()};
+      for (const Symbol &symbol : GetSymbolVector(*rhs)) {
+        if (first == symbol) {
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+namespace operation {
+template <typename T> Expr<SomeType> AsSomeExpr(const T &x) {
+  return AsGenericExpr(common::Clone(x));
+}
+
+template <bool IgnoreResizingConverts>
+struct ArgumentExtractor
+    : public Traverse<ArgumentExtractor<IgnoreResizingConverts>,
+          std::pair<operation::Operator, std::vector<Expr<SomeType>>>, false> {
+  using Arguments = std::vector<Expr<SomeType>>;
+  using Result = std::pair<operation::Operator, Arguments>;
+  using Base =
+      Traverse<ArgumentExtractor<IgnoreResizingConverts>, Result, false>;
+  static constexpr auto IgnoreResizes{IgnoreResizingConverts};
+  static constexpr auto Logical{common::TypeCategory::Logical};
+  ArgumentExtractor() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <int Kind>
+  Result operator()(const Constant<Type<Logical, Kind>> &x) const {
+    if (const auto &val{x.GetScalarValue()}) {
+      return val->IsTrue()
+          ? std::make_pair(operation::Operator::True, Arguments{})
+          : std::make_pair(operation::Operator::False, Arguments{});
+    }
+    return Default();
+  }
+
+  template <typename R> Result operator()(const FunctionRef<R> &x) const {
+    Result result{operation::OperationCode(x.proc()), {}};
+    for (size_t i{0}, e{x.arguments().size()}; i != e; ++i) {
+      if (auto *e{x.UnwrapArgExpr(i)}) {
+        result.second.push_back(*e);
+      }
+    }
+    return result;
+  }
+
+  template <typename D, typename R, typename... Os>
+  Result operator()(const Operation<D, R, Os...> &x) const {
+    if constexpr (std::is_same_v<D, Parentheses<R>>) {
+      // Ignore top-level parentheses.
+      return (*this)(x.template operand<0>());
+    }
+    if constexpr (IgnoreResizes && std::is_same_v<D, Convert<R, R::category>>) {
+      // Ignore conversions within the same category.
+      // Atomic operations on int(kind=1) may be implicitly widened
+      // to int(kind=4) for example.
+      return (*this)(x.template operand<0>());
+    } else {
+      return std::make_pair(operation::OperationCode(x),
+          OperationArgs(x, std::index_sequence_for<Os...>{}));
+    }
+  }
+
+  template <typename T> Result operator()(const Designator<T> &x) const {
+    return {operation::Operator::Identity, {AsSomeExpr(x)}};
+  }
+
+  template <typename T> Result operator()(const Constant<T> &x) const {
+    return {operation::Operator::Identity, {AsSomeExpr(x)}};
+  }
+
+  template <typename... Rs>
+  Result Combine(Result &&result, Rs &&...results) const {
+    // There shouldn't be any combining needed, since we're stopping the
+    // traversal at the top-level operation, but implement one that picks
+    // the first non-empty result.
+    if constexpr (sizeof...(Rs) == 0) {
+      return std::move(result);
+    } else {
+      if (!result.second.empty()) {
+        return std::move(result);
+      } else {
+        return Combine(std::move(results)...);
+      }
+    }
+  }
+
+private:
+  template <typename D, typename R, typename... Os, size_t... Is>
+  Arguments OperationArgs(
+      const Operation<D, R, Os...> &x, std::index_sequence<Is...>) const {
+    return Arguments{Expr<SomeType>(x.template operand<Is>())...};
+  }
+};
+} // namespace operation
+
+std::string operation::ToString(operation::Operator op) {
+  switch (op) {
+  case Operator::Unknown:
+    return "??";
+  case Operator::Add:
+    return "+";
+  case Operator::And:
+    return "AND";
+  case Operator::Associated:
+    return "ASSOCIATED";
+  case Operator::Call:
+    return "function-call";
+  case Operator::Constant:
+    return "constant";
+  case Operator::Convert:
+    return "type-conversion";
+  case Operator::Div:
+    return "/";
+  case Operator::Eq:
+    return "==";
+  case Operator::Eqv:
+    return "EQV";
+  case Operator::False:
+    return ".FALSE.";
+  case Operator::Ge:
+    return ">=";
+  case Operator::Gt:
+    return ">";
+  case Operator::Identity:
+    return "identity";
+  case Operator::Intrinsic:
+    return "intrinsic";
+  case Operator::Le:
+    return "<=";
+  case Operator::Lt:
+    return "<";
+  case Operator::Max:
+    return "MAX";
+  case Operator::Min:
+    return "MIN";
+  case Operator::Mul:
+    return "*";
+  case Operator::Ne:
+    return "/=";
+  case Operator::Neqv:
+    return "NEQV/EOR";
+  case Operator::Not:
+    return "NOT";
+  case Operator::Or:
+    return "OR";
+  case Operator::Pow:
+    return "**";
+  case Operator::Resize:
+    return "resize";
+  case Operator::Sub:
+    return "-";
+  case Operator::True:
+    return ".TRUE.";
+  }
+  llvm_unreachable("Unhandler operator");
+}
+
+operation::Operator operation::OperationCode(const ProcedureDesignator &proc) {
+  Operator code{llvm::StringSwitch<Operator>(proc.GetName())
+          .Case("associated", Operator::Associated)
+          .Case("min", Operator::Min)
+          .Case("max", Operator::Max)
+          .Case("iand", Operator::And)
+          .Case("ior", Operator::Or)
+          .Case("ieor", Operator::Neqv)
+          .Default(Operator::Call)};
+  if (code == Operator::Call && proc.GetSpecificIntrinsic()) {
+    return Operator::Intrinsic;
+  }
+  return code;
+}
+
+std::pair<operation::Operator, std::vector<Expr<SomeType>>>
+GetTopLevelOperation(const Expr<SomeType> &expr) {
+  return operation::ArgumentExtractor<true>{}(expr);
+}
+
+namespace operation {
+struct ConvertCollector
+    : public Traverse<ConvertCollector,
+          std::pair<std::optional<Expr<SomeType>>, std::vector<DynamicType>>,
+          false> {
+  using Result =
+      std::pair<std::optional<Expr<SomeType>>, std::vector<DynamicType>>;
+  using Base = Traverse<ConvertCollector, Result, false>;
+  ConvertCollector() : Base(*this) {}
+
+  Result Default() const { return {}; }
+
+  using Base::operator();
+
+  template <typename T> Result operator()(const Designator<T> &x) const {
+    return {AsSomeExpr(x), {}};
+  }
+
+  template <typename T> Result operator()(const FunctionRef<T> &x) const {
+    return {AsSomeExpr(x), {}};
+  }
+
+  template <typename T> Result operator()(const Constant<T> &x) const {
+    return {AsSomeExpr(x), {}};
+  }
+
+  template <typename D, typename R, typename... Os>
+  Result operator()(const Operation<D, R, Os...> &x) const {
+    if constexpr (std::is_same_v<D, Parentheses<R>>) {
+      // Ignore parentheses.
+      return (*this)(x.template operand<0>());
+    } else if constexpr (is_convert_v<D>) {
+      // Convert should always have a typed result, so it should be safe to
+      // dereference x.GetType().
+      return Combine(
+          {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
+    } else if constexpr (is_complex_constructor_v<D>) {
+      // This is a conversion iff the imaginary operand is 0.
+      if (IsZero(x.template operand<1>())) {
+        return Combine(
+            {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
+      } else {
+        return {AsSomeExpr(x.derived()), {}};
+      }
+    } else {
+      return {AsSomeExpr(x.derived()), {}};
+    }
+  }
+
+  template <typename... Rs>
+  Result Combine(Result &&result, Rs &&...results) const {
+    Result v(std::move(result));
+    auto setValue{[](std::optional<Expr<SomeType>> &x,
+                      std::optional<Expr<SomeType>> &&y) {
+      assert((!x.has_value() || !y.has_value()) && "Multiple designators");
+      if (!x.has_value()) {
+        x = std::move(y);
+      }
+    }};
+    auto moveAppend{[](auto &accum, auto &&other) {
+      for (auto &&s : other) {
+        accum.push_back(std::move(s));
+      }
+    }};
+    (setValue(v.first, std::move(results).first), ...);
+    (moveAppend(v.second, std::move(results).second), ...);
+    return v;
+  }
+
+private:
+  template <typename A> static bool IsZero(const A &x) { return false; }
+  template <typename T> static bool IsZero(const Expr<T> &x) {
+    return common::visit([](auto &&s) { return IsZero(s); }, x.u);
+  }
+  template <typename T> static bool IsZero(const Constant<T> &x) {
+    if (auto &&maybeScalar{x.GetScalarValue()}) {
+      return maybeScalar->IsZero();
+    } else {
+      return false;
+    }
+  }
+
+  template <typename T> struct is_convert {
+    static constexpr bool value{false};
+  };
+  template <typename T, common::TypeCategory C>
+  struct is_convert<Convert<T, C>> {
+    static constexpr bool value{true};
+  };
+  template <int K> struct is_convert<ComplexComponent<K>> {
+    // Conversion from complex to real.
+    static constexpr bool value{true};
+  };
+  template <typename T>
+  static constexpr bool is_convert_v{is_convert<T>::value};
+
+  template <typename T> struct is_complex_constructor {
+    static constexpr bool value{false};
+  };
+  template <int K> struct is_complex_constructor<ComplexConstructor<K>> {
+    static constexpr bool value{true};
+  };
+  template <typename T>
+  static constexpr bool is_complex_constructor_v{
+      is_complex_constructor<T>::value};
+};
+} // namespace operation
+
+std::optional<Expr<SomeType>> GetConvertInput(const Expr<SomeType> &x) {
+  // This returns Expr<SomeType>{x} when x is a designator/functionref/constant.
+  return operation::ConvertCollector{}(x).first;
+}
+
+bool IsSameOrConvertOf(const Expr<SomeType> &expr, const Expr<SomeType> &x) {
+  // Check if expr is same as x, or a sequence of Convert operations on x.
+  if (expr == x) {
+    return true;
+  } else if (auto maybe{GetConvertInput(expr)}) {
+    return *maybe == x;
+  } else {
+    return false;
+  }
+}
 } // namespace Fortran::evaluate
 
 namespace Fortran::semantics {
diff --git a/flang/lib/Lower/OpenACC.cpp b/flang/lib/Lower/OpenACC.cpp
index 69e9c53baa74..3ef3330cba2d 100644
--- a/flang/lib/Lower/OpenACC.cpp
+++ b/flang/lib/Lower/OpenACC.cpp
@@ -654,7 +654,7 @@ void genAtomicCapture(Fortran::lower::AbstractConverter &converter,
   mlir::Block &block = atomicCaptureOp->getRegion(0).back();
   firOpBuilder.setInsertionPointToStart(&block);
   if (Fortran::parser::CheckForSingleVariableOnRHS(stmt1)) {
-    if (Fortran::semantics::CheckForSymbolMatch(
+    if (Fortran::evaluate::CheckForSymbolMatch(
             Fortran::semantics::GetExpr(stmt2Var),
             Fortran::semantics::GetExpr(stmt2Expr))) {
       // Atomic capture construct is of the form [capture-stmt, update-stmt]
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 2595a08f626e..1c51fd60d570 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2934,11 +2934,12 @@ genAtomicUpdate(lower::AbstractConverter &converter,
   mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
 
   // This must exist by now.
-  SomeExpr input = *semantics::GetConvertInput(assign.rhs);
-  std::vector<SomeExpr> args{semantics::GetTopLevelOperation(input).second};
+  SomeExpr input = *Fortran::evaluate::GetConvertInput(assign.rhs);
+  std::vector<SomeExpr> args{
+      Fortran::evaluate::GetTopLevelOperation(input).second};
   assert(!args.empty() && "Update operation without arguments");
   for (auto &arg : args) {
-    if (!semantics::IsSameOrConvertOf(arg, atom)) {
+    if (!Fortran::evaluate::IsSameOrConvertOf(arg, atom)) {
       mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
       overrides.try_emplace(&arg, val);
     }
diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp
index 68cea6739830..3abb5a304b00 100644
--- a/flang/lib/Semantics/check-omp-structure.cpp
+++ b/flang/lib/Semantics/check-omp-structure.cpp
@@ -12,6 +12,7 @@
 #include "flang/Evaluate/check-expression.h"
 #include "flang/Evaluate/expression.h"
 #include "flang/Evaluate/shape.h"
+#include "flang/Evaluate/tools.h"
 #include "flang/Evaluate/type.h"
 #include "flang/Parser/parse-tree.h"
 #include "flang/Semantics/expression.h"
@@ -2987,6 +2988,8 @@ static bool IsPointerAssignment(const evaluate::Assignment &x) {
       std::holds_alternative<evaluate::Assignment::BoundsRemapping>(x.u);
 }
 
+namespace operation = Fortran::evaluate::operation;
+
 static bool IsCheckForAssociated(const SomeExpr &cond) {
   return GetTopLevelOperation(cond).first == operation::Operator::Associated;
 }
diff --git a/flang/lib/Semantics/tools.cpp b/flang/lib/Semantics/tools.cpp
index bf520d04a50c..d053179448c0 100644
--- a/flang/lib/Semantics/tools.cpp
+++ b/flang/lib/Semantics/tools.cpp
@@ -17,7 +17,6 @@
 #include "flang/Semantics/tools.h"
 #include "flang/Semantics/type.h"
 #include "flang/Support/Fortran.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <set>
@@ -1789,332 +1788,4 @@ bool HadUseError(
   }
 }
 
-bool CheckForSymbolMatch(const SomeExpr *lhs, const SomeExpr *rhs) {
-  if (lhs && rhs) {
-    if (SymbolVector lhsSymbols{evaluate::GetSymbolVector(*lhs)};
-        !lhsSymbols.empty()) {
-      const Symbol &first{*lhsSymbols.front()};
-      for (const Symbol &symbol : evaluate::GetSymbolVector(*rhs)) {
-        if (first == symbol) {
-          return true;
-        }
-      }
-    }
-  }
-  return false;
-}
-
-namespace operation {
-template <typename T> //
-SomeExpr asSomeExpr(const T &x) {
-  auto copy{x};
-  return AsGenericExpr(std::move(copy));
-}
-
-template <bool IgnoreResizingConverts> //
-struct ArgumentExtractor
-    : public evaluate::Traverse<ArgumentExtractor<IgnoreResizingConverts>,
-          std::pair<operation::Operator, std::vector<SomeExpr>>, false> {
-  using Arguments = std::vector<SomeExpr>;
-  using Result = std::pair<operation::Operator, Arguments>;
-  using Base = evaluate::Traverse<ArgumentExtractor<IgnoreResizingConverts>,
-      Result, false>;
-  static constexpr auto IgnoreResizes = IgnoreResizingConverts;
-  static constexpr auto Logical = common::TypeCategory::Logical;
-  ArgumentExtractor() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <int Kind> //
-  Result operator()(
-      const evaluate::Constant<evaluate::Type<Logical, Kind>> &x) const {
-    if (const auto &val{x.GetScalarValue()}) {
-      return val->IsTrue()
-          ? std::make_pair(operation::Operator::True, Arguments{})
-          : std::make_pair(operation::Operator::False, Arguments{});
-    }
-    return Default();
-  }
-
-  template <typename R> //
-  Result operator()(const evaluate::FunctionRef<R> &x) const {
-    Result result{operation::OperationCode(x.proc()), {}};
-    for (size_t i{0}, e{x.arguments().size()}; i != e; ++i) {
-      if (auto *e{x.UnwrapArgExpr(i)}) {
-        result.second.push_back(*e);
-      }
-    }
-    return result;
-  }
-
-  template <typename D, typename R, typename... Os>
-  Result operator()(const evaluate::Operation<D, R, Os...> &x) const {
-    if constexpr (std::is_same_v<D, evaluate::Parentheses<R>>) {
-      // Ignore top-level parentheses.
-      return (*this)(x.template operand<0>());
-    }
-    if constexpr (IgnoreResizes &&
-        std::is_same_v<D, evaluate::Convert<R, R::category>>) {
-      // Ignore conversions within the same category.
-      // Atomic operations on int(kind=1) may be implicitly widened
-      // to int(kind=4) for example.
-      return (*this)(x.template operand<0>());
-    } else {
-      return std::make_pair(operation::OperationCode(x),
-          OperationArgs(x, std::index_sequence_for<Os...>{}));
-    }
-  }
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    return {operation::Operator::Identity, {asSomeExpr(x)}};
-  }
-
-  template <typename T> //
-  Result operator()(const evaluate::Constant<T> &x) const {
-    return {operation::Operator::Identity, {asSomeExpr(x)}};
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    // There shouldn't be any combining needed, since we're stopping the
-    // traversal at the top-level operation, but implement one that picks
-    // the first non-empty result.
-    if constexpr (sizeof...(Rs) == 0) {
-      return std::move(result);
-    } else {
-      if (!result.second.empty()) {
-        return std::move(result);
-      } else {
-        return Combine(std::move(results)...);
-      }
-    }
-  }
-
-private:
-  template <typename D, typename R, typename... Os, size_t... Is>
-  Arguments OperationArgs(const evaluate::Operation<D, R, Os...> &x,
-      std::index_sequence<Is...>) const {
-    return Arguments{SomeExpr(x.template operand<Is>())...};
-  }
-};
-} // namespace operation
-
-std::string operation::ToString(operation::Operator op) {
-  switch (op) {
-  case Operator::Unknown:
-    return "??";
-  case Operator::Add:
-    return "+";
-  case Operator::And:
-    return "AND";
-  case Operator::Associated:
-    return "ASSOCIATED";
-  case Operator::Call:
-    return "function-call";
-  case Operator::Constant:
-    return "constant";
-  case Operator::Convert:
-    return "type-conversion";
-  case Operator::Div:
-    return "/";
-  case Operator::Eq:
-    return "==";
-  case Operator::Eqv:
-    return "EQV";
-  case Operator::False:
-    return ".FALSE.";
-  case Operator::Ge:
-    return ">=";
-  case Operator::Gt:
-    return ">";
-  case Operator::Identity:
-    return "identity";
-  case Operator::Intrinsic:
-    return "intrinsic";
-  case Operator::Le:
-    return "<=";
-  case Operator::Lt:
-    return "<";
-  case Operator::Max:
-    return "MAX";
-  case Operator::Min:
-    return "MIN";
-  case Operator::Mul:
-    return "*";
-  case Operator::Ne:
-    return "/=";
-  case Operator::Neqv:
-    return "NEQV/EOR";
-  case Operator::Not:
-    return "NOT";
-  case Operator::Or:
-    return "OR";
-  case Operator::Pow:
-    return "**";
-  case Operator::Resize:
-    return "resize";
-  case Operator::Sub:
-    return "-";
-  case Operator::True:
-    return ".TRUE.";
-  }
-  llvm_unreachable("Unhandler operator");
-}
-
-operation::Operator operation::OperationCode(
-    const evaluate::ProcedureDesignator &proc) {
-  Operator code = llvm::StringSwitch<Operator>(proc.GetName())
-                      .Case("associated", Operator::Associated)
-                      .Case("min", Operator::Min)
-                      .Case("max", Operator::Max)
-                      .Case("iand", Operator::And)
-                      .Case("ior", Operator::Or)
-                      .Case("ieor", Operator::Neqv)
-                      .Default(Operator::Call);
-  if (code == Operator::Call && proc.GetSpecificIntrinsic()) {
-    return Operator::Intrinsic;
-  }
-  return code;
-}
-
-std::pair<operation::Operator, std::vector<SomeExpr>> GetTopLevelOperation(
-    const SomeExpr &expr) {
-  return operation::ArgumentExtractor<true>{}(expr);
-}
-
-namespace operation {
-struct ConvertCollector
-    : public evaluate::Traverse<ConvertCollector,
-          std::pair<MaybeExpr, std::vector<evaluate::DynamicType>>, false> {
-  using Result = std::pair<MaybeExpr, std::vector<evaluate::DynamicType>>;
-  using Base = evaluate::Traverse<ConvertCollector, Result, false>;
-  ConvertCollector() : Base(*this) {}
-
-  Result Default() const { return {}; }
-
-  using Base::operator();
-
-  template <typename T> //
-  Result operator()(const evaluate::Designator<T> &x) const {
-    return {asSomeExpr(x), {}};
-  }
-
-  template <typename T> //
-  Result operator()(const evaluate::FunctionRef<T> &x) const {
-    return {asSomeExpr(x), {}};
-  }
-
-  template <typename T> //
-  Result operator()(const evaluate::Constant<T> &x) const {
-    return {asSomeExpr(x), {}};
-  }
-
-  template <typename D, typename R, typename... Os>
-  Result operator()(const evaluate::Operation<D, R, Os...> &x) const {
-    if constexpr (std::is_same_v<D, evaluate::Parentheses<R>>) {
-      // Ignore parentheses.
-      return (*this)(x.template operand<0>());
-    } else if constexpr (is_convert_v<D>) {
-      // Convert should always have a typed result, so it should be safe to
-      // dereference x.GetType().
-      return Combine(
-          {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
-    } else if constexpr (is_complex_constructor_v<D>) {
-      // This is a conversion iff the imaginary operand is 0.
-      if (IsZero(x.template operand<1>())) {
-        return Combine(
-            {std::nullopt, {*x.GetType()}}, (*this)(x.template operand<0>()));
-      } else {
-        return {asSomeExpr(x.derived()), {}};
-      }
-    } else {
-      return {asSomeExpr(x.derived()), {}};
-    }
-  }
-
-  template <typename... Rs> //
-  Result Combine(Result &&result, Rs &&...results) const {
-    Result v(std::move(result));
-    auto setValue{[](MaybeExpr &x, MaybeExpr &&y) {
-      assert((!x.has_value() || !y.has_value()) && "Multiple designators");
-      if (!x.has_value()) {
-        x = std::move(y);
-      }
-    }};
-    auto moveAppend{[](auto &accum, auto &&other) {
-      for (auto &&s : other) {
-        accum.push_back(std::move(s));
-      }
-    }};
-    (setValue(v.first, std::move(results).first), ...);
-    (moveAppend(v.second, std::move(results).second), ...);
-    return v;
-  }
-
-private:
-  template <typename T> //
-  static bool IsZero(const T &x) {
-    return false;
-  }
-  template <typename T> //
-  static bool IsZero(const evaluate::Expr<T> &x) {
-    return common::visit([](auto &&s) { return IsZero(s); }, x.u);
-  }
-  template <typename T> //
-  static bool IsZero(const evaluate::Constant<T> &x) {
-    if (auto &&maybeScalar{x.GetScalarValue()}) {
-      return maybeScalar->IsZero();
-    } else {
-      return false;
-    }
-  }
-
-  template <typename T> //
-  struct is_convert {
-    static constexpr bool value{false};
-  };
-  template <typename T, common::TypeCategory C> //
-  struct is_convert<evaluate::Convert<T, C>> {
-    static constexpr bool value{true};
-  };
-  template <int K> //
-  struct is_convert<evaluate::ComplexComponent<K>> {
-    // Conversion from complex to real.
-    static constexpr bool value{true};
-  };
-  template <typename T> //
-  static constexpr bool is_convert_v = is_convert<T>::value;
-
-  template <typename T> //
-  struct is_complex_constructor {
-    static constexpr bool value{false};
-  };
-  template <int K> //
-  struct is_complex_constructor<evaluate::ComplexConstructor<K>> {
-    static constexpr bool value{true};
-  };
-  template <typename T> //
-  static constexpr bool is_complex_constructor_v =
-      is_complex_constructor<T>::value;
-};
-} // namespace operation
-
-MaybeExpr GetConvertInput(const SomeExpr &x) {
-  // This returns SomeExpr(x) when x is a designator/functionref/constant.
-  return operation::ConvertCollector{}(x).first;
-}
-
-bool IsSameOrConvertOf(const SomeExpr &expr, const SomeExpr &x) {
-  // Check if expr is same as x, or a sequence of Convert operations on x.
-  if (expr == x) {
-    return true;
-  } else if (auto maybe{GetConvertInput(expr)}) {
-    return *maybe == x;
-  } else {
-    return false;
-  }
-}
 } // namespace Fortran::semantics
\ No newline at end of file
-- 
cgit v1.2.3


From 53336ad488c953d41e744d42873d712276be980f Mon Sep 17 00:00:00 2001
From: Ross Brunton <ross@codeplay.com>
Date: Thu, 19 Jun 2025 22:02:03 +0100
Subject: [Offload] Move (most) global state to an `OffloadContext` struct
 (#144494)

Rather than having a number of static local variables, we now use
a single `OffloadContext` struct to store global state. This is
initialised by `olInit`, but is never deleted (de-initialization of
Offload isn't yet implemented).

The error reporting mechanism has not been moved to the struct, since
that's going to cause issues with teardown (error messages must outlive
liboffload).
---
 offload/liboffload/include/OffloadImpl.hpp     | 12 ++--
 offload/liboffload/src/OffloadImpl.cpp         | 82 ++++++++++++++++----------
 offload/liboffload/src/OffloadLib.cpp          |  5 --
 offload/tools/offload-tblgen/EntryPointGen.cpp | 37 +++++++-----
 4 files changed, 80 insertions(+), 56 deletions(-)

diff --git a/offload/liboffload/include/OffloadImpl.hpp b/offload/liboffload/include/OffloadImpl.hpp
index 9b0a21cb9ae1..a12d8c47a180 100644
--- a/offload/liboffload/include/OffloadImpl.hpp
+++ b/offload/liboffload/include/OffloadImpl.hpp
@@ -22,12 +22,12 @@
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Error.h"
 
-struct OffloadConfig {
-  bool TracingEnabled = false;
-  bool ValidationEnabled = true;
-};
-
-OffloadConfig &offloadConfig();
+namespace llvm {
+namespace offload {
+bool isTracingEnabled();
+bool isValidationEnabled();
+} // namespace offload
+} // namespace llvm
 
 // Use the StringSet container to efficiently deduplicate repeated error
 // strings (e.g. if the same error is hit constantly in a long running program)
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 770c212d804d..f02497c0a633 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -93,22 +93,36 @@ struct AllocInfo {
   ol_alloc_type_t Type;
 };
 
-using AllocInfoMapT = DenseMap<void *, AllocInfo>;
-AllocInfoMapT &allocInfoMap() {
-  static AllocInfoMapT AllocInfoMap{};
-  return AllocInfoMap;
-}
+// Global shared state for liboffload
+struct OffloadContext;
+static OffloadContext *OffloadContextVal;
+struct OffloadContext {
+  OffloadContext(OffloadContext &) = delete;
+  OffloadContext(OffloadContext &&) = delete;
+  OffloadContext &operator=(OffloadContext &) = delete;
+  OffloadContext &operator=(OffloadContext &&) = delete;
+
+  bool TracingEnabled = false;
+  bool ValidationEnabled = true;
+  DenseMap<void *, AllocInfo> AllocInfoMap{};
+  SmallVector<ol_platform_impl_t, 4> Platforms{};
+
+  ol_device_handle_t HostDevice() {
+    // The host platform is always inserted last
+    return &Platforms.back().Devices[0];
+  }
 
-using PlatformVecT = SmallVector<ol_platform_impl_t, 4>;
-PlatformVecT &Platforms() {
-  static PlatformVecT Platforms;
-  return Platforms;
-}
+  static OffloadContext &get() {
+    assert(OffloadContextVal);
+    return *OffloadContextVal;
+  }
+};
 
-ol_device_handle_t HostDevice() {
-  // The host platform is always inserted last
-  return &Platforms().back().Devices[0];
+// If the context is uninited, then we assume tracing is disabled
+bool isTracingEnabled() {
+  return OffloadContextVal && OffloadContext::get().TracingEnabled;
 }
+bool isValidationEnabled() { return OffloadContext::get().ValidationEnabled; }
 
 template <typename HandleT> Error olDestroy(HandleT Handle) {
   delete Handle;
@@ -130,10 +144,12 @@ constexpr ol_platform_backend_t pluginNameToBackend(StringRef Name) {
 #include "Shared/Targets.def"
 
 void initPlugins() {
+  auto *Context = new OffloadContext{};
+
   // Attempt to create an instance of each supported plugin.
 #define PLUGIN_TARGET(Name)                                                    \
   do {                                                                         \
-    Platforms().emplace_back(ol_platform_impl_t{                               \
+    Context->Platforms.emplace_back(ol_platform_impl_t{                        \
         std::unique_ptr<GenericPluginTy>(createPlugin_##Name()),               \
         {},                                                                    \
         pluginNameToBackend(#Name)});                                          \
@@ -141,7 +157,7 @@ void initPlugins() {
 #include "Shared/Targets.def"
 
   // Preemptively initialize all devices in the plugin
-  for (auto &Platform : Platforms()) {
+  for (auto &Platform : Context->Platforms) {
     // Do not use the host plugin - it isn't supported.
     if (Platform.BackendType == OL_PLATFORM_BACKEND_UNKNOWN)
       continue;
@@ -157,15 +173,16 @@ void initPlugins() {
   }
 
   // Add the special host device
-  auto &HostPlatform = Platforms().emplace_back(
+  auto &HostPlatform = Context->Platforms.emplace_back(
       ol_platform_impl_t{nullptr,
                          {ol_device_impl_t{-1, nullptr, nullptr}},
                          OL_PLATFORM_BACKEND_HOST});
-  HostDevice()->Platform = &HostPlatform;
+  Context->HostDevice()->Platform = &HostPlatform;
+
+  Context->TracingEnabled = std::getenv("OFFLOAD_TRACE");
+  Context->ValidationEnabled = !std::getenv("OFFLOAD_DISABLE_VALIDATION");
 
-  offloadConfig().TracingEnabled = std::getenv("OFFLOAD_TRACE");
-  offloadConfig().ValidationEnabled =
-      !std::getenv("OFFLOAD_DISABLE_VALIDATION");
+  OffloadContextVal = Context;
 }
 
 // TODO: We can properly reference count here and manage the resources in a more
@@ -229,7 +246,7 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
 
   // Find the info if it exists under any of the given names
   auto GetInfo = [&](std::vector<std::string> Names) {
-    if (Device == HostDevice())
+    if (Device == OffloadContext::get().HostDevice())
       return std::string("Host");
 
     if (!Device->Device)
@@ -251,8 +268,9 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
   case OL_DEVICE_INFO_PLATFORM:
     return ReturnValue(Device->Platform);
   case OL_DEVICE_INFO_TYPE:
-    return Device == HostDevice() ? ReturnValue(OL_DEVICE_TYPE_HOST)
-                                  : ReturnValue(OL_DEVICE_TYPE_GPU);
+    return Device == OffloadContext::get().HostDevice()
+               ? ReturnValue(OL_DEVICE_TYPE_HOST)
+               : ReturnValue(OL_DEVICE_TYPE_GPU);
   case OL_DEVICE_INFO_NAME:
     return ReturnValue(GetInfo({"Device Name"}).c_str());
   case OL_DEVICE_INFO_VENDOR:
@@ -280,7 +298,7 @@ Error olGetDeviceInfoSize_impl(ol_device_handle_t Device,
 }
 
 Error olIterateDevices_impl(ol_device_iterate_cb_t Callback, void *UserData) {
-  for (auto &Platform : Platforms()) {
+  for (auto &Platform : OffloadContext::get().Platforms) {
     for (auto &Device : Platform.Devices) {
       if (!Callback(&Device, UserData)) {
         break;
@@ -311,16 +329,17 @@ Error olMemAlloc_impl(ol_device_handle_t Device, ol_alloc_type_t Type,
     return Alloc.takeError();
 
   *AllocationOut = *Alloc;
-  allocInfoMap().insert_or_assign(*Alloc, AllocInfo{Device, Type});
+  OffloadContext::get().AllocInfoMap.insert_or_assign(*Alloc,
+                                                      AllocInfo{Device, Type});
   return Error::success();
 }
 
 Error olMemFree_impl(void *Address) {
-  if (!allocInfoMap().contains(Address))
+  if (!OffloadContext::get().AllocInfoMap.contains(Address))
     return createOffloadError(ErrorCode::INVALID_ARGUMENT,
                               "address is not a known allocation");
 
-  auto AllocInfo = allocInfoMap().at(Address);
+  auto AllocInfo = OffloadContext::get().AllocInfoMap.at(Address);
   auto Device = AllocInfo.Device;
   auto Type = AllocInfo.Type;
 
@@ -328,7 +347,7 @@ Error olMemFree_impl(void *Address) {
           Device->Device->dataDelete(Address, convertOlToPluginAllocTy(Type)))
     return Res;
 
-  allocInfoMap().erase(Address);
+  OffloadContext::get().AllocInfoMap.erase(Address);
 
   return Error::success();
 }
@@ -395,7 +414,8 @@ Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
                     ol_device_handle_t DstDevice, const void *SrcPtr,
                     ol_device_handle_t SrcDevice, size_t Size,
                     ol_event_handle_t *EventOut) {
-  if (DstDevice == HostDevice() && SrcDevice == HostDevice()) {
+  auto Host = OffloadContext::get().HostDevice();
+  if (DstDevice == Host && SrcDevice == Host) {
     if (!Queue) {
       std::memcpy(DstPtr, SrcPtr, Size);
       return Error::success();
@@ -410,11 +430,11 @@ Error olMemcpy_impl(ol_queue_handle_t Queue, void *DstPtr,
   // If no queue is given the memcpy will be synchronous
   auto QueueImpl = Queue ? Queue->AsyncInfo : nullptr;
 
-  if (DstDevice == HostDevice()) {
+  if (DstDevice == Host) {
     if (auto Res =
             SrcDevice->Device->dataRetrieve(DstPtr, SrcPtr, Size, QueueImpl))
       return Res;
-  } else if (SrcDevice == HostDevice()) {
+  } else if (SrcDevice == Host) {
     if (auto Res =
             DstDevice->Device->dataSubmit(DstPtr, SrcPtr, Size, QueueImpl))
       return Res;
diff --git a/offload/liboffload/src/OffloadLib.cpp b/offload/liboffload/src/OffloadLib.cpp
index 8662d3a44124..0a65815e5969 100644
--- a/offload/liboffload/src/OffloadLib.cpp
+++ b/offload/liboffload/src/OffloadLib.cpp
@@ -30,11 +30,6 @@ ol_code_location_t *&currentCodeLocation() {
   return CodeLoc;
 }
 
-OffloadConfig &offloadConfig() {
-  static OffloadConfig Config{};
-  return Config;
-}
-
 namespace llvm {
 namespace offload {
 // Pull in the declarations for the implementation functions. The actual entry
diff --git a/offload/tools/offload-tblgen/EntryPointGen.cpp b/offload/tools/offload-tblgen/EntryPointGen.cpp
index 85c5c50bf2f2..13aa0d1f6318 100644
--- a/offload/tools/offload-tblgen/EntryPointGen.cpp
+++ b/offload/tools/offload-tblgen/EntryPointGen.cpp
@@ -35,21 +35,30 @@ static void EmitValidationFunc(const FunctionRec &F, raw_ostream &OS) {
   }
   OS << ") {\n";
 
-  OS << TAB_1 "if (offloadConfig().ValidationEnabled) {\n";
-  // Emit validation checks
-  for (const auto &Return : F.getReturns()) {
-    for (auto &Condition : Return.getConditions()) {
-      if (Condition.starts_with("`") && Condition.ends_with("`")) {
-        auto ConditionString = Condition.substr(1, Condition.size() - 2);
-        OS << formatv(TAB_2 "if ({0}) {{\n", ConditionString);
-        OS << formatv(TAB_3 "return createOffloadError(error::ErrorCode::{0}, "
-                            "\"validation failure: {1}\");\n",
-                      Return.getUnprefixedValue(), ConditionString);
-        OS << TAB_2 "}\n\n";
+  bool HasValidation = llvm::any_of(F.getReturns(), [](auto &R) {
+    return llvm::any_of(R.getConditions(), [](auto &C) {
+      return C.starts_with("`") && C.ends_with("`");
+    });
+  });
+
+  if (HasValidation) {
+    OS << TAB_1 "if (llvm::offload::isValidationEnabled()) {\n";
+    // Emit validation checks
+    for (const auto &Return : F.getReturns()) {
+      for (auto &Condition : Return.getConditions()) {
+        if (Condition.starts_with("`") && Condition.ends_with("`")) {
+          auto ConditionString = Condition.substr(1, Condition.size() - 2);
+          OS << formatv(TAB_2 "if ({0}) {{\n", ConditionString);
+          OS << formatv(TAB_3
+                        "return createOffloadError(error::ErrorCode::{0}, "
+                        "\"validation failure: {1}\");\n",
+                        Return.getUnprefixedValue(), ConditionString);
+          OS << TAB_2 "}\n\n";
+        }
       }
     }
+    OS << TAB_1 "}\n\n";
   }
-  OS << TAB_1 "}\n\n";
 
   // Perform actual function call to the implementation
   ParamNameList = ParamNameList.substr(0, ParamNameList.size() - 2);
@@ -74,7 +83,7 @@ static void EmitEntryPointFunc(const FunctionRec &F, raw_ostream &OS) {
   OS << ") {\n";
 
   // Emit pre-call prints
-  OS << TAB_1 "if (offloadConfig().TracingEnabled) {\n";
+  OS << TAB_1 "if (llvm::offload::isTracingEnabled()) {\n";
   OS << formatv(TAB_2 "llvm::errs() << \"---> {0}\";\n", F.getName());
   OS << TAB_1 "}\n\n";
 
@@ -85,7 +94,7 @@ static void EmitEntryPointFunc(const FunctionRec &F, raw_ostream &OS) {
       PrefixLower, F.getName(), ParamNameList);
 
   // Emit post-call prints
-  OS << TAB_1 "if (offloadConfig().TracingEnabled) {\n";
+  OS << TAB_1 "if (llvm::offload::isTracingEnabled()) {\n";
   if (F.getParams().size() > 0) {
     OS << formatv(TAB_2 "{0} Params = {{", F.getParamStructName());
     for (const auto &Param : F.getParams()) {
-- 
cgit v1.2.3


From 2f3a8fd0b3322baac25e5595313413ed4cd1158f Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Thu, 19 Jun 2025 17:06:52 -0400
Subject: [libc] Skip x87 floating point register and only update mxcsr for
 x86_64 targets when raising exceptions inside math functions. (#144951)

Updating x87 floating point register significantly affect the
performance of the functions.
All the floating point exception reads will merge the results from both
mxcsr and x87 registers anyway.
---
 libc/src/__support/FPUtil/FEnvImpl.h        |  5 +++++
 libc/src/__support/FPUtil/x86_64/FEnvImpl.h | 13 ++++++++-----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/libc/src/__support/FPUtil/FEnvImpl.h b/libc/src/__support/FPUtil/FEnvImpl.h
index ba145a3da45c..76910880eb81 100644
--- a/libc/src/__support/FPUtil/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/FEnvImpl.h
@@ -91,7 +91,12 @@ LIBC_INLINE static int set_except_if_required([[maybe_unused]] int excepts) {
 LIBC_INLINE static int raise_except_if_required([[maybe_unused]] int excepts) {
 #ifndef LIBC_MATH_HAS_NO_EXCEPT
   if (math_errhandling & MATH_ERREXCEPT)
+#ifdef LIBC_TARGET_ARCH_IS_X86_64
+    return raise_except</*SKIP_X87_FPU*/ true>(excepts);
+#else  // !LIBC_TARGET_ARCH_IS_X86
     return raise_except(excepts);
+#endif // LIBC_TARGET_ARCH_IS_X86
+
 #endif // LIBC_MATH_HAS_NO_EXCEPT
   return 0;
 }
diff --git a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
index b77178ea69ea..560727c22978 100644
--- a/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
+++ b/libc/src/__support/FPUtil/x86_64/FEnvImpl.h
@@ -239,7 +239,7 @@ LIBC_INLINE int set_except(int excepts) {
   return 0;
 }
 
-LIBC_INLINE int raise_except(int excepts) {
+template <bool SKIP_X87_FPU = false> LIBC_INLINE int raise_except(int excepts) {
   uint16_t status_value = internal::get_status_value_for_except(excepts);
 
   // We set the status flag for exception one at a time and call the
@@ -256,13 +256,16 @@ LIBC_INLINE int raise_except(int excepts) {
   // when raising the next exception.
 
   auto raise_helper = [](uint16_t singleExceptFlag) {
-    internal::X87StateDescriptor state;
+    if constexpr (!SKIP_X87_FPU) {
+      internal::X87StateDescriptor state;
+      internal::get_x87_state_descriptor(state);
+      state.status_word |= singleExceptFlag;
+      internal::write_x87_state_descriptor(state);
+    }
+
     uint32_t mxcsr = 0;
-    internal::get_x87_state_descriptor(state);
     mxcsr = internal::get_mxcsr();
-    state.status_word |= singleExceptFlag;
     mxcsr |= singleExceptFlag;
-    internal::write_x87_state_descriptor(state);
     internal::write_mxcsr(mxcsr);
     internal::fwait();
   };
-- 
cgit v1.2.3


From 53ea522d1b87c144a1faeffea62d50a4d9907a38 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 19 Jun 2025 14:12:14 -0700
Subject: [LV] Introduce and use VPBuilder::createScalarZExtOrTrunc [nfc]
 (#144946)

Reduce redundant code, make the flow slightly easier to read.
---
 .../Vectorize/LoopVectorizationPlanner.h           | 11 ++++++
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp  | 40 ++++++++--------------
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index 70f541d64b30..d17c64a778e8 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -278,6 +278,17 @@ public:
         new VPInstructionWithType(Opcode, Op, ResultTy, {}, DL));
   }
 
+  VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
+                                   DebugLoc DL) {
+    if (ResultTy == SrcTy)
+      return Op;
+    Instruction::CastOps CastOp =
+        ResultTy->getScalarSizeInBits() < SrcTy->getScalarSizeInBits()
+            ? Instruction::Trunc
+            : Instruction::ZExt;
+    return createScalarCast(CastOp, Op, ResultTy, DL);
+  }
+
   VPWidenCastRecipe *createWidenCast(Instruction::CastOps Opcode, VPValue *Op,
                                      Type *ResultTy) {
     return tryInsertInstruction(new VPWidenCastRecipe(Opcode, Op, ResultTy));
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index cc73ae44f9c0..60a8837fb76a 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -797,15 +797,8 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
   VPValue *FirstActiveLane =
       B.createNaryOp(VPInstruction::FirstActiveLane, Mask, DL);
   Type *FirstActiveLaneType = TypeInfo.inferScalarType(FirstActiveLane);
-  if (CanonicalIVType != FirstActiveLaneType) {
-    Instruction::CastOps CastOp =
-        CanonicalIVType->getScalarSizeInBits() <
-                FirstActiveLaneType->getScalarSizeInBits()
-            ? Instruction::Trunc
-            : Instruction::ZExt;
-    FirstActiveLane =
-        B.createScalarCast(CastOp, FirstActiveLane, CanonicalIVType, DL);
-  }
+  FirstActiveLane = B.createScalarZExtOrTrunc(FirstActiveLane, CanonicalIVType,
+                                              FirstActiveLaneType, DL);
   EndValue = B.createNaryOp(Instruction::Add, {EndValue, FirstActiveLane}, DL);
 
   // `getOptimizableIVOf()` always returns the pre-incremented IV, so if it
@@ -2182,13 +2175,10 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
     VPValue *MaxEVL = &Plan.getVF();
     // Emit VPScalarCastRecipe in preheader if VF is not a 32 bits integer.
     VPBuilder Builder(LoopRegion->getPreheaderVPBB());
-    if (unsigned VFSize =
-            TypeInfo.inferScalarType(MaxEVL)->getScalarSizeInBits();
-        VFSize != 32) {
-      MaxEVL = Builder.createScalarCast(
-          VFSize > 32 ? Instruction::Trunc : Instruction::ZExt, MaxEVL,
-          Type::getInt32Ty(Ctx), DebugLoc());
-    }
+    MaxEVL = Builder.createScalarZExtOrTrunc(MaxEVL, Type::getInt32Ty(Ctx),
+                                             TypeInfo.inferScalarType(MaxEVL),
+                                             DebugLoc());
+
     Builder.setInsertPoint(Header, Header->getFirstNonPhi());
     PrevEVL = Builder.createScalarPhi({MaxEVL, &EVL}, DebugLoc(), "prev.evl");
   }
@@ -2286,6 +2276,7 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
     return false;
 
   auto *CanonicalIVPHI = Plan.getCanonicalIV();
+  auto *CanIVTy = CanonicalIVPHI->getScalarType();
   VPValue *StartV = CanonicalIVPHI->getStartValue();
 
   // Create the ExplicitVectorLengthPhi recipe in the main loop.
@@ -2297,8 +2288,8 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
       Instruction::Sub, {Plan.getTripCount(), EVLPhi}, DebugLoc(), "avl");
   if (MaxSafeElements) {
     // Support for MaxSafeDist for correct loop emission.
-    VPValue *AVLSafe = Plan.getOrAddLiveIn(
-        ConstantInt::get(CanonicalIVPHI->getScalarType(), *MaxSafeElements));
+    VPValue *AVLSafe =
+        Plan.getOrAddLiveIn(ConstantInt::get(CanIVTy, *MaxSafeElements));
     VPValue *Cmp = Builder.createICmp(ICmpInst::ICMP_ULT, AVL, AVLSafe);
     AVL = Builder.createSelect(Cmp, AVL, AVLSafe, DebugLoc(), "safe_avl");
   }
@@ -2308,13 +2299,12 @@ bool VPlanTransforms::tryAddExplicitVectorLength(
   auto *CanonicalIVIncrement =
       cast<VPInstruction>(CanonicalIVPHI->getBackedgeValue());
   Builder.setInsertPoint(CanonicalIVIncrement);
-  VPSingleDefRecipe *OpVPEVL = VPEVL;
-  if (unsigned IVSize = CanonicalIVPHI->getScalarType()->getScalarSizeInBits();
-      IVSize != 32) {
-    OpVPEVL = Builder.createScalarCast(
-        IVSize < 32 ? Instruction::Trunc : Instruction::ZExt, OpVPEVL,
-        CanonicalIVPHI->getScalarType(), CanonicalIVIncrement->getDebugLoc());
-  }
+  VPValue *OpVPEVL = VPEVL;
+
+  auto *I32Ty = Type::getInt32Ty(CanIVTy->getContext());
+  OpVPEVL = Builder.createScalarZExtOrTrunc(
+      OpVPEVL, CanIVTy, I32Ty, CanonicalIVIncrement->getDebugLoc());
+
   auto *NextEVLIV = Builder.createOverflowingOp(
       Instruction::Add, {OpVPEVL, EVLPhi},
       {CanonicalIVIncrement->hasNoUnsignedWrap(),
-- 
cgit v1.2.3


From d3a2931d8af87c20aaede991acda0b5f313075c3 Mon Sep 17 00:00:00 2001
From: lntue <lntue@google.com>
Date: Thu, 19 Jun 2025 17:51:01 -0400
Subject: [libc] Use `raise_except_if_required` for log2f. (#144961)

---
 libc/src/math/generic/log2f.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp
index b25ec41f277b..cff718eec216 100644
--- a/libc/src/math/generic/log2f.cpp
+++ b/libc/src/math/generic/log2f.cpp
@@ -79,7 +79,7 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) {
     }
     if (xbits.is_neg() && !xbits.is_nan()) {
       fputil::set_errno_if_required(EDOM);
-      fputil::raise_except(FE_INVALID);
+      fputil::raise_except_if_required(FE_INVALID);
       return FPBits::quiet_nan().get_val();
     }
     if (xbits.is_inf_or_nan()) {
-- 
cgit v1.2.3


From a9d175f1735a508ac05ab48d83a99071ba97c10e Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 14:57:58 -0700
Subject: [CodeGen] Use range-based for loops (NFC) (#144939)

---
 clang/lib/CodeGen/CGBlocks.cpp       |  6 +++---
 clang/lib/CodeGen/CGCleanup.cpp      |  4 ++--
 clang/lib/CodeGen/CGDeclCXX.cpp      |  6 +++---
 clang/lib/CodeGen/CGException.cpp    | 10 +++++-----
 clang/lib/CodeGen/CGExpr.cpp         | 12 ++++++------
 clang/lib/CodeGen/CGExprConstant.cpp |  4 +---
 clang/lib/CodeGen/CGObjC.cpp         |  4 ++--
 clang/lib/CodeGen/CGObjCGNU.cpp      | 19 ++++++++-----------
 clang/lib/CodeGen/CGObjCMac.cpp      | 20 ++++++++------------
 clang/lib/CodeGen/CGObjCRuntime.cpp  |  4 +---
 10 files changed, 39 insertions(+), 50 deletions(-)

diff --git a/clang/lib/CodeGen/CGBlocks.cpp b/clang/lib/CodeGen/CGBlocks.cpp
index 729758ddce56..f3ddf7bf9a46 100644
--- a/clang/lib/CodeGen/CGBlocks.cpp
+++ b/clang/lib/CodeGen/CGBlocks.cpp
@@ -1415,10 +1415,10 @@ llvm::Function *CodeGenFunction::GenerateBlockFunction(
   // Arrange for local static and local extern declarations to appear
   // to be local to this function as well, in case they're directly
   // referenced in a block.
-  for (DeclMapTy::const_iterator i = ldm.begin(), e = ldm.end(); i != e; ++i) {
-    const auto *var = dyn_cast<VarDecl>(i->first);
+  for (const auto &KV : ldm) {
+    const auto *var = dyn_cast<VarDecl>(KV.first);
     if (var && !var->hasLocalStorage())
-      setAddrOfLocalVar(var, i->second);
+      setAddrOfLocalVar(var, KV.second);
   }
 
   // Begin building the function declaration.
diff --git a/clang/lib/CodeGen/CGCleanup.cpp b/clang/lib/CodeGen/CGCleanup.cpp
index 4ed2c5183c47..28ac9bf39635 100644
--- a/clang/lib/CodeGen/CGCleanup.cpp
+++ b/clang/lib/CodeGen/CGCleanup.cpp
@@ -962,8 +962,8 @@ void CodeGenFunction::PopCleanupBlock(bool FallthroughIsBranchThrough,
 
       // Append the prepared cleanup prologue from above.
       llvm::BasicBlock *NormalExit = Builder.GetInsertBlock();
-      for (unsigned I = 0, E = InstsToAppend.size(); I != E; ++I)
-        InstsToAppend[I]->insertInto(NormalExit, NormalExit->end());
+      for (llvm::Instruction *Inst : InstsToAppend)
+        Inst->insertInto(NormalExit, NormalExit->end());
 
       // Optimistically hope that any fixups will continue falling through.
       for (unsigned I = FixupDepth, E = EHStack.getNumBranchFixups();
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
index 69d77f283db3..7ae99935c8ad 100644
--- a/clang/lib/CodeGen/CGDeclCXX.cpp
+++ b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -1121,9 +1121,9 @@ CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
       EmitObjCAutoreleasePoolCleanup(token);
     }
 
-    for (unsigned i = 0, e = Decls.size(); i != e; ++i)
-      if (Decls[i])
-        EmitRuntimeCall(Decls[i]);
+    for (llvm::Function *Decl : Decls)
+      if (Decl)
+        EmitRuntimeCall(Decl);
 
     Scope.ForceCleanup();
 
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index e0367282355c..ad138b9876e8 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -319,9 +319,9 @@ static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
     llvm::Function *F = dyn_cast<llvm::Function>(U);
     if (!F) return false;
 
-    for (auto BB = F->begin(), E = F->end(); BB != E; ++BB) {
-      if (BB->isLandingPad())
-        if (!LandingPadHasOnlyCXXUses(BB->getLandingPadInst()))
+    for (llvm::BasicBlock &BB : *F) {
+      if (BB.isLandingPad())
+        if (!LandingPadHasOnlyCXXUses(BB.getLandingPadInst()))
           return false;
     }
   }
@@ -937,8 +937,8 @@ llvm::BasicBlock *CodeGenFunction::EmitLandingPad() {
                              filterTypes[0]->getType() : Int8PtrTy,
                            filterTypes.size());
 
-    for (unsigned i = 0, e = filterTypes.size(); i != e; ++i)
-      Filters.push_back(cast<llvm::Constant>(filterTypes[i]));
+    for (llvm::Value *filterType : filterTypes)
+      Filters.push_back(cast<llvm::Constant>(filterType));
     llvm::Constant *FilterArray = llvm::ConstantArray::get(AType, Filters);
     LPadInst->addClause(FilterArray);
 
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 6cb348ffdf55..85c768807572 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -3801,8 +3801,8 @@ void CodeGenFunction::EmitCheck(
       ArgTypes.push_back(Args.back()->getType());
     }
 
-    for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
-      Args.push_back(EmitCheckValue(DynamicArgs[i]));
+    for (llvm::Value *DynamicArg : DynamicArgs) {
+      Args.push_back(EmitCheckValue(DynamicArg));
       ArgTypes.push_back(IntPtrTy);
     }
   }
@@ -4932,8 +4932,8 @@ EmitExtVectorElementExpr(const ExtVectorElementExpr *E) {
   llvm::Constant *BaseElts = Base.getExtVectorElts();
   SmallVector<llvm::Constant *, 4> CElts;
 
-  for (unsigned i = 0, e = Indices.size(); i != e; ++i)
-    CElts.push_back(BaseElts->getAggregateElement(Indices[i]));
+  for (unsigned Index : Indices)
+    CElts.push_back(BaseElts->getAggregateElement(Index));
   llvm::Constant *CV = llvm::ConstantVector::get(CElts);
   return LValue::MakeExtVectorElt(Base.getExtVectorAddress(), CV, type,
                                   Base.getBaseInfo(), TBAAAccessInfo());
@@ -6660,8 +6660,8 @@ static LValueOrRValue emitPseudoObjectExpr(CodeGenFunction &CGF,
   }
 
   // Unbind all the opaques now.
-  for (unsigned i = 0, e = opaques.size(); i != e; ++i)
-    opaques[i].unbind(CGF);
+  for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
+    opaque.unbind(CGF);
 
   return result;
 }
diff --git a/clang/lib/CodeGen/CGExprConstant.cpp b/clang/lib/CodeGen/CGExprConstant.cpp
index 069cc09cd91d..715bd392f59f 100644
--- a/clang/lib/CodeGen/CGExprConstant.cpp
+++ b/clang/lib/CodeGen/CGExprConstant.cpp
@@ -871,9 +871,7 @@ bool ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
     }
     llvm::stable_sort(Bases);
 
-    for (unsigned I = 0, N = Bases.size(); I != N; ++I) {
-      BaseInfo &Base = Bases[I];
-
+    for (const BaseInfo &Base : Bases) {
       bool IsPrimaryBase = Layout.getPrimaryBase() == Base.Decl;
       Build(Val.getStructBase(Base.Index), Base.Decl, IsPrimaryBase,
             VTableClass, Offset + Base.Offset);
diff --git a/clang/lib/CodeGen/CGObjC.cpp b/clang/lib/CodeGen/CGObjC.cpp
index 8a1b44a0c157..6f87444d3f67 100644
--- a/clang/lib/CodeGen/CGObjC.cpp
+++ b/clang/lib/CodeGen/CGObjC.cpp
@@ -3173,8 +3173,8 @@ ARCExprEmitter<Impl,Result>::visitPseudoObjectExpr(const PseudoObjectExpr *E) {
   }
 
   // Unbind all the opaques now.
-  for (unsigned i = 0, e = opaques.size(); i != e; ++i)
-    opaques[i].unbind(CGF);
+  for (CodeGenFunction::OpaqueValueMappingData &opaque : opaques)
+    opaque.unbind(CGF);
 
   return result;
 }
diff --git a/clang/lib/CodeGen/CGObjCGNU.cpp b/clang/lib/CodeGen/CGObjCGNU.cpp
index 3fc837c12a92..d828702cbb87 100644
--- a/clang/lib/CodeGen/CGObjCGNU.cpp
+++ b/clang/lib/CodeGen/CGObjCGNU.cpp
@@ -1103,8 +1103,7 @@ class CGObjCGNUstep2 : public CGObjCGNUstep {
     bool isNamed = !isNonASCII;
     if (isNamed) {
       StringName = ".objc_str_";
-      for (int i=0,e=Str.size() ; i<e ; ++i) {
-        unsigned char c = Str[i];
+      for (unsigned char c : Str) {
         if (isalnum(c))
           StringName += c;
         else if (c == ' ')
@@ -2560,10 +2559,9 @@ llvm::Value *CGObjCGNU::GetTypedSelector(CodeGenFunction &CGF, Selector Sel,
   SmallVectorImpl<TypedSelector> &Types = SelectorTable[Sel];
   llvm::GlobalAlias *SelValue = nullptr;
 
-  for (SmallVectorImpl<TypedSelector>::iterator i = Types.begin(),
-      e = Types.end() ; i!=e ; i++) {
-    if (i->first == TypeEncoding) {
-      SelValue = i->second;
+  for (const TypedSelector &Type : Types) {
+    if (Type.first == TypeEncoding) {
+      SelValue = Type.second;
       break;
     }
   }
@@ -3333,13 +3331,12 @@ CGObjCGNU::GenerateProtocolList(ArrayRef<std::string> Protocols) {
   ProtocolList.addInt(LongTy, Protocols.size());
 
   auto Elements = ProtocolList.beginArray(PtrToInt8Ty);
-  for (const std::string *iter = Protocols.begin(), *endIter = Protocols.end();
-      iter != endIter ; iter++) {
+  for (const std::string &Protocol : Protocols) {
     llvm::Constant *protocol = nullptr;
-    llvm::StringMap<llvm::Constant*>::iterator value =
-      ExistingProtocols.find(*iter);
+    llvm::StringMap<llvm::Constant *>::iterator value =
+        ExistingProtocols.find(Protocol);
     if (value == ExistingProtocols.end()) {
-      protocol = GenerateEmptyProtocol(*iter);
+      protocol = GenerateEmptyProtocol(Protocol);
     } else {
       protocol = value->getValue();
     }
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index 1c23a8b4db91..a52c92cdbc83 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -2702,8 +2702,8 @@ llvm::Constant *CGObjCCommonMac::getBitmapBlockLayout(bool ComputeByrefLayout) {
   unsigned char inst = (BLOCK_LAYOUT_OPERATOR << 4) | 0;
   Layout.push_back(inst);
   std::string BitMap;
-  for (unsigned i = 0, e = Layout.size(); i != e; i++)
-    BitMap += Layout[i];
+  for (unsigned char C : Layout)
+    BitMap += C;
 
   if (CGM.getLangOpts().ObjCGCBitmapPrint) {
     if (ComputeByrefLayout)
@@ -4225,9 +4225,8 @@ FragileHazards::FragileHazards(CodeGenFunction &CGF) : CGF(CGF) {
     return;
 
   // Collect all the blocks in the function.
-  for (llvm::Function::iterator I = CGF.CurFn->begin(), E = CGF.CurFn->end();
-       I != E; ++I)
-    BlocksBeforeTry.insert(&*I);
+  for (llvm::BasicBlock &BB : *CGF.CurFn)
+    BlocksBeforeTry.insert(&BB);
 
   llvm::FunctionType *AsmFnTy = GetAsmFnType();
 
@@ -4299,9 +4298,7 @@ void FragileHazards::emitHazardsInNewBlocks() {
   CGBuilderTy Builder(CGF, CGF.getLLVMContext());
 
   // Iterate through all blocks, skipping those prior to the try.
-  for (llvm::Function::iterator FI = CGF.CurFn->begin(), FE = CGF.CurFn->end();
-       FI != FE; ++FI) {
-    llvm::BasicBlock &BB = *FI;
+  for (llvm::BasicBlock &BB : *CGF.CurFn) {
     if (BlocksBeforeTry.count(&BB))
       continue;
 
@@ -4348,10 +4345,9 @@ void FragileHazards::collectLocals() {
   // Collect all the allocas currently in the function.  This is
   // probably way too aggressive.
   llvm::BasicBlock &Entry = CGF.CurFn->getEntryBlock();
-  for (llvm::BasicBlock::iterator I = Entry.begin(), E = Entry.end(); I != E;
-       ++I)
-    if (isa<llvm::AllocaInst>(*I) && !AllocasToIgnore.count(&*I))
-      Locals.push_back(&*I);
+  for (llvm::Instruction &I : Entry)
+    if (isa<llvm::AllocaInst>(I) && !AllocasToIgnore.count(&I))
+      Locals.push_back(&I);
 }
 
 llvm::FunctionType *FragileHazards::GetAsmFnType() {
diff --git a/clang/lib/CodeGen/CGObjCRuntime.cpp b/clang/lib/CodeGen/CGObjCRuntime.cpp
index dfb0fd14d93a..6e2f32022a01 100644
--- a/clang/lib/CodeGen/CGObjCRuntime.cpp
+++ b/clang/lib/CodeGen/CGObjCRuntime.cpp
@@ -220,9 +220,7 @@ void CGObjCRuntime::EmitTryCatchStmt(CodeGenFunction &CGF,
   CGBuilderTy::InsertPoint SavedIP = CGF.Builder.saveAndClearIP();
 
   // Emit the handlers.
-  for (unsigned I = 0, E = Handlers.size(); I != E; ++I) {
-    CatchHandler &Handler = Handlers[I];
-
+  for (CatchHandler &Handler : Handlers) {
     CGF.EmitBlock(Handler.Block);
 
     CodeGenFunction::LexicalScope Cleanups(CGF, Handler.Body->getSourceRange());
-- 
cgit v1.2.3


From a8edda195c9fe0d48ee7f6f2438c9575ebbad7f2 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 14:58:05 -0700
Subject: [llvm] Remove unused includes (NFC) (#144941)

These are identified by misc-include-cleaner.  I've filtered out those
that break builds.  Also, I'm staying away from llvm-config.h,
config.h, and Compiler.h, which likely cause platform- or
compiler-specific build failures.
---
 llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp      | 2 --
 llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp | 2 --
 llvm/lib/Linker/IRMover.cpp                                   | 1 -
 llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp               | 1 -
 4 files changed, 6 deletions(-)

diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp
index ecf5c0e519ca..4a5d5a8ffab6 100644
--- a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp
@@ -12,8 +12,6 @@
 
 #include "COFFDirectiveParser.h"
 
-#include <array>
-
 using namespace llvm;
 using namespace jitlink;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp
index 4482eedc0070..178c7ee1c01b 100644
--- a/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.cpp
@@ -8,10 +8,8 @@
 
 #include "llvm/ExecutionEngine/Orc/UnwindInfoRegistrationPlugin.h"
 
-#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ExecutionEngine/Orc/Shared/MachOObjectFormat.h"
 #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h"
-#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Module.h"
 
 #define DEBUG_TYPE "orc"
diff --git a/llvm/lib/Linker/IRMover.cpp b/llvm/lib/Linker/IRMover.cpp
index 2a9709050162..a466ce5bf0d4 100644
--- a/llvm/lib/Linker/IRMover.cpp
+++ b/llvm/lib/Linker/IRMover.cpp
@@ -9,7 +9,6 @@
 #include "llvm/Linker/IRMover.h"
 #include "LinkDiagnosticInfo.h"
 #include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/Constants.h"
diff --git a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
index 222eb19e3eee..ea9155c60d38 100644
--- a/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
+++ b/llvm/lib/Target/BPF/BPFPreserveStaticOffset.cpp
@@ -108,7 +108,6 @@
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
-#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
-- 
cgit v1.2.3


From dfb5cadf5e816e542e46d3f0551b6a148a93ce3d Mon Sep 17 00:00:00 2001
From: Koakuma <koachan@protonmail.com>
Date: Fri, 20 Jun 2025 05:12:51 +0700
Subject: [SPARC][IAS] Properly set implied feature sets for ISA
 levels/extensions (#143232)

Some SPARC ISA levels and/or extensions are defined in a way such that
the availability of it implies the availability of other, more fundamental
ISA features (for example, targeting 64-bit environment implies that
V9 instructions are available).
Properly set those in the TableGen definitions.

Fixes https://github.com/llvm/llvm-project/issues/142388.
---
 llvm/lib/Target/Sparc/Sparc.td                     | 18 +++--
 llvm/lib/Target/Sparc/SparcInstrInfo.td            |  3 +-
 llvm/test/CodeGen/SPARC/ctlz.ll                    | 42 +++++-------
 llvm/test/CodeGen/SPARC/cttz.ll                    | 46 ++++++-------
 llvm/test/CodeGen/SPARC/inlineasm-v9.ll            |  9 +++
 llvm/test/CodeGen/SPARC/inlineasm.ll               |  9 ---
 .../MC/Sparc/Relocations/relocation-specifier.s    | 77 ++++++++++++----------
 llvm/test/MC/Sparc/sparcv9-instructions.s          | 21 ++++++
 8 files changed, 121 insertions(+), 104 deletions(-)

diff --git a/llvm/lib/Target/Sparc/Sparc.td b/llvm/lib/Target/Sparc/Sparc.td
index 6e6c887e60e1..8588d2d28b71 100644
--- a/llvm/lib/Target/Sparc/Sparc.td
+++ b/llvm/lib/Target/Sparc/Sparc.td
@@ -42,22 +42,28 @@ def FeatureV8Deprecated
                      "Enable deprecated V8 instructions in V9 mode">;
 def FeatureVIS
   : SubtargetFeature<"vis", "IsVIS", "true",
-                     "Enable UltraSPARC Visual Instruction Set extensions">;
+                     "Enable UltraSPARC Visual Instruction Set extensions",
+                     [FeatureV9]>;
 def FeatureVIS2
   : SubtargetFeature<"vis2", "IsVIS2", "true",
-                     "Enable Visual Instruction Set extensions II">;
+                     "Enable Visual Instruction Set extensions II",
+                     [FeatureV9]>;
 def FeatureVIS3
   : SubtargetFeature<"vis3", "IsVIS3", "true",
-                     "Enable Visual Instruction Set extensions III">;
+                     "Enable Visual Instruction Set extensions III",
+                     [FeatureV9]>;
 def FeatureUA2005
   : SubtargetFeature<"ua2005", "IsUA2005", "true",
-                     "Enable UltraSPARC Architecture 2005 extensions">;
+                     "Enable UltraSPARC Architecture 2005 extensions",
+                     [FeatureV9, FeatureVIS, FeatureVIS2]>;
 def FeatureUA2007
   : SubtargetFeature<"ua2007", "IsUA2007", "true",
-                     "Enable UltraSPARC Architecture 2007 extensions">;
+                     "Enable UltraSPARC Architecture 2007 extensions",
+                     [FeatureV9, FeatureVIS, FeatureVIS2]>;
 def FeatureOSA2011
   : SubtargetFeature<"osa2011", "IsOSA2011", "true",
-                     "Enable Oracle SPARC Architecture 2011 extensions">;
+                     "Enable Oracle SPARC Architecture 2011 extensions",
+                     [FeatureV9, FeatureVIS, FeatureVIS2, FeatureVIS3]>;
 def FeatureLeon
   : SubtargetFeature<"leon", "IsLeon", "true",
                      "Enable LEON extensions">;
diff --git a/llvm/lib/Target/Sparc/SparcInstrInfo.td b/llvm/lib/Target/Sparc/SparcInstrInfo.td
index 074a04a5dd74..1be017be1c64 100644
--- a/llvm/lib/Target/Sparc/SparcInstrInfo.td
+++ b/llvm/lib/Target/Sparc/SparcInstrInfo.td
@@ -24,7 +24,8 @@ include "SparcInstrFormats.td"
 def Is32Bit : Predicate<"!Subtarget->is64Bit()">;
 
 // True when generating 64-bit code. This also implies HasV9.
-def Is64Bit : Predicate<"Subtarget->is64Bit()">;
+def Is64Bit : Predicate<"Subtarget->is64Bit()">,
+              AssemblerPredicate<(all_of FeatureV9)>;
 
 def UseSoftMulDiv : Predicate<"Subtarget->useSoftMulDiv()">,
               AssemblerPredicate<(all_of FeatureSoftMulDiv)>;
diff --git a/llvm/test/CodeGen/SPARC/ctlz.ll b/llvm/test/CodeGen/SPARC/ctlz.ll
index 75930190f516..f7dc309452b3 100644
--- a/llvm/test/CodeGen/SPARC/ctlz.ll
+++ b/llvm/test/CodeGen/SPARC/ctlz.ll
@@ -207,20 +207,15 @@ define i64 @i64_nopoison(i64 %x) nounwind {
 ;
 ; SPARC-VIS3-LABEL: i64_nopoison:
 ; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o2
+; SPARC-VIS3-NEXT:    lzcnt %o2, %o2
+; SPARC-VIS3-NEXT:    add %o2, -32, %o2
+; SPARC-VIS3-NEXT:    srl %o1, 0, %o1
+; SPARC-VIS3-NEXT:    lzcnt %o1, %o1
+; SPARC-VIS3-NEXT:    add %o1, -32, %o1
+; SPARC-VIS3-NEXT:    add %o1, 32, %o1
 ; SPARC-VIS3-NEXT:    cmp %o0, 0
-; SPARC-VIS3-NEXT:    bne .LBB2_2
-; SPARC-VIS3-NEXT:    nop
-; SPARC-VIS3-NEXT:  ! %bb.1:
-; SPARC-VIS3-NEXT:    srl %o1, 0, %o0
-; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
-; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    add %o0, 32, %o1
-; SPARC-VIS3-NEXT:    retl
-; SPARC-VIS3-NEXT:    mov %g0, %o0
-; SPARC-VIS3-NEXT:  .LBB2_2:
-; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
-; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
-; SPARC-VIS3-NEXT:    add %o0, -32, %o1
+; SPARC-VIS3-NEXT:    movne %icc, %o2, %o1
 ; SPARC-VIS3-NEXT:    retl
 ; SPARC-VIS3-NEXT:    mov %g0, %o0
 ;
@@ -311,20 +306,15 @@ define i64 @i64_poison(i64 %x) nounwind {
 ;
 ; SPARC-VIS3-LABEL: i64_poison:
 ; SPARC-VIS3:       ! %bb.0:
+; SPARC-VIS3-NEXT:    srl %o0, 0, %o2
+; SPARC-VIS3-NEXT:    lzcnt %o2, %o2
+; SPARC-VIS3-NEXT:    add %o2, -32, %o2
+; SPARC-VIS3-NEXT:    srl %o1, 0, %o1
+; SPARC-VIS3-NEXT:    lzcnt %o1, %o1
+; SPARC-VIS3-NEXT:    add %o1, -32, %o1
+; SPARC-VIS3-NEXT:    add %o1, 32, %o1
 ; SPARC-VIS3-NEXT:    cmp %o0, 0
-; SPARC-VIS3-NEXT:    bne .LBB3_2
-; SPARC-VIS3-NEXT:    nop
-; SPARC-VIS3-NEXT:  ! %bb.1:
-; SPARC-VIS3-NEXT:    srl %o1, 0, %o0
-; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
-; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    add %o0, 32, %o1
-; SPARC-VIS3-NEXT:    retl
-; SPARC-VIS3-NEXT:    mov %g0, %o0
-; SPARC-VIS3-NEXT:  .LBB3_2:
-; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
-; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
-; SPARC-VIS3-NEXT:    add %o0, -32, %o1
+; SPARC-VIS3-NEXT:    movne %icc, %o2, %o1
 ; SPARC-VIS3-NEXT:    retl
 ; SPARC-VIS3-NEXT:    mov %g0, %o0
 ;
diff --git a/llvm/test/CodeGen/SPARC/cttz.ll b/llvm/test/CodeGen/SPARC/cttz.ll
index edabd7d560ed..138f67dbf684 100644
--- a/llvm/test/CodeGen/SPARC/cttz.ll
+++ b/llvm/test/CodeGen/SPARC/cttz.ll
@@ -254,28 +254,25 @@ define i64 @i64_nopoison(i64 %x) nounwind {
 ;
 ; SPARC-VIS3-LABEL: i64_nopoison:
 ; SPARC-VIS3:       ! %bb.0:
-; SPARC-VIS3-NEXT:    cmp %o1, 0
-; SPARC-VIS3-NEXT:    bne .LBB2_2
-; SPARC-VIS3-NEXT:    nop
-; SPARC-VIS3-NEXT:  ! %bb.1:
-; SPARC-VIS3-NEXT:    add %o0, -1, %o1
-; SPARC-VIS3-NEXT:    andn %o1, %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -1, %o2
+; SPARC-VIS3-NEXT:    andn %o2, %o0, %o0
 ; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
 ; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
 ; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    ba .LBB2_3
-; SPARC-VIS3-NEXT:    mov 64, %o1
-; SPARC-VIS3-NEXT:  .LBB2_2:
+; SPARC-VIS3-NEXT:    mov 64, %o2
+; SPARC-VIS3-NEXT:    sub %o2, %o0, %o2
 ; SPARC-VIS3-NEXT:    add %o1, -1, %o0
 ; SPARC-VIS3-NEXT:    andn %o0, %o1, %o0
 ; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
 ; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
 ; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    mov 32, %o1
-; SPARC-VIS3-NEXT:  .LBB2_3:
-; SPARC-VIS3-NEXT:    sub %o1, %o0, %o1
-; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov 32, %o3
+; SPARC-VIS3-NEXT:    sub %o3, %o0, %o0
+; SPARC-VIS3-NEXT:    cmp %o1, 0
+; SPARC-VIS3-NEXT:    movne %icc, %o0, %o2
 ; SPARC-VIS3-NEXT:    mov %g0, %o0
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %o2, %o1
 ;
 ; SPARC64-LABEL: i64_nopoison:
 ; SPARC64:       ! %bb.0:
@@ -376,28 +373,25 @@ define i64 @i64_poison(i64 %x) nounwind {
 ;
 ; SPARC-VIS3-LABEL: i64_poison:
 ; SPARC-VIS3:       ! %bb.0:
-; SPARC-VIS3-NEXT:    cmp %o1, 0
-; SPARC-VIS3-NEXT:    bne .LBB3_2
-; SPARC-VIS3-NEXT:    nop
-; SPARC-VIS3-NEXT:  ! %bb.1:
-; SPARC-VIS3-NEXT:    add %o0, -1, %o1
-; SPARC-VIS3-NEXT:    andn %o1, %o0, %o0
+; SPARC-VIS3-NEXT:    add %o0, -1, %o2
+; SPARC-VIS3-NEXT:    andn %o2, %o0, %o0
 ; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
 ; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
 ; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    ba .LBB3_3
-; SPARC-VIS3-NEXT:    mov 64, %o1
-; SPARC-VIS3-NEXT:  .LBB3_2:
+; SPARC-VIS3-NEXT:    mov 64, %o2
+; SPARC-VIS3-NEXT:    sub %o2, %o0, %o2
 ; SPARC-VIS3-NEXT:    add %o1, -1, %o0
 ; SPARC-VIS3-NEXT:    andn %o0, %o1, %o0
 ; SPARC-VIS3-NEXT:    srl %o0, 0, %o0
 ; SPARC-VIS3-NEXT:    lzcnt %o0, %o0
 ; SPARC-VIS3-NEXT:    add %o0, -32, %o0
-; SPARC-VIS3-NEXT:    mov 32, %o1
-; SPARC-VIS3-NEXT:  .LBB3_3:
-; SPARC-VIS3-NEXT:    sub %o1, %o0, %o1
-; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov 32, %o3
+; SPARC-VIS3-NEXT:    sub %o3, %o0, %o0
+; SPARC-VIS3-NEXT:    cmp %o1, 0
+; SPARC-VIS3-NEXT:    movne %icc, %o0, %o2
 ; SPARC-VIS3-NEXT:    mov %g0, %o0
+; SPARC-VIS3-NEXT:    retl
+; SPARC-VIS3-NEXT:    mov %o2, %o1
 ;
 ; SPARC64-LABEL: i64_poison:
 ; SPARC64:       ! %bb.0:
diff --git a/llvm/test/CodeGen/SPARC/inlineasm-v9.ll b/llvm/test/CodeGen/SPARC/inlineasm-v9.ll
index 47126d5d64da..289bb33fb864 100644
--- a/llvm/test/CodeGen/SPARC/inlineasm-v9.ll
+++ b/llvm/test/CodeGen/SPARC/inlineasm-v9.ll
@@ -58,3 +58,12 @@ Entry:
   tail call void asm sideeffect "", "{o0}"(i64 %val)
   ret void
 }
+
+; CHECK-LABEL: test_twinword:
+; CHECK: rd  %pc, %i1
+; CHECK: srlx %i1, 32, %i0
+
+define i64 @test_twinword(){
+  %1 = tail call i64 asm sideeffect "rd %asr5, ${0:L} \0A\09 srlx ${0:L}, 32, ${0:H}", "={i0}"()
+  ret i64 %1
+}
diff --git a/llvm/test/CodeGen/SPARC/inlineasm.ll b/llvm/test/CodeGen/SPARC/inlineasm.ll
index 3ca2168efb71..07411385bdf3 100644
--- a/llvm/test/CodeGen/SPARC/inlineasm.ll
+++ b/llvm/test/CodeGen/SPARC/inlineasm.ll
@@ -144,15 +144,6 @@ entry:
   ret void
 }
 
-; CHECK-LABEL: test_twinword:
-; CHECK: rd  %asr5, %i1
-; CHECK: srlx %i1, 32, %i0
-
-define i64 @test_twinword(){
-  %1 = tail call i64 asm sideeffect "rd %asr5, ${0:L} \0A\09 srlx ${0:L}, 32, ${0:H}", "={i0}"()
-  ret i64 %1
-}
-
 ; CHECK-LABEL: test_symbol:
 ; CHECK: ba,a brtarget
 define void @test_symbol() {
diff --git a/llvm/test/MC/Sparc/Relocations/relocation-specifier.s b/llvm/test/MC/Sparc/Relocations/relocation-specifier.s
index 1d89babb5e6c..8a996c99e55a 100644
--- a/llvm/test/MC/Sparc/Relocations/relocation-specifier.s
+++ b/llvm/test/MC/Sparc/Relocations/relocation-specifier.s
@@ -1,20 +1,21 @@
 # RUN: llvm-mc %s -triple=sparc | FileCheck %s --check-prefix=ASM
-# RUN: llvm-mc %s -triple=sparcv9 | FileCheck %s --check-prefix=ASM
+# RUN: llvm-mc %s --defsym V9=1 -triple=sparcv9 | FileCheck %s --check-prefixes=ASM,ASM-V9
 
 # RUN: llvm-mc %s -triple=sparc -filetype=obj -o %t
 # RUN: llvm-objdump -dr %t | FileCheck %s --check-prefix=OBJDUMP
-# RUN: llvm-mc %s -triple=sparcv9 -filetype=obj -o %t
-# RUN: llvm-objdump -dr %t | FileCheck %s --check-prefix=OBJDUMP
 # RUN: llvm-readelf -s - < %t | FileCheck %s --check-prefix=READELF --implicit-check-not=TLS
+# RUN: llvm-mc %s --defsym V9=1 -triple=sparcv9 -filetype=obj -o %t
+# RUN: llvm-objdump -dr %t | FileCheck %s --check-prefixes=OBJDUMP,OBJDUMP-V9
+# RUN: llvm-readelf -s - < %t | FileCheck %s --check-prefixes=READELF,READELF-V9 --implicit-check-not=TLS
 
 # READELF: TLS     LOCAL  DEFAULT [[#]] s_tle_hix22
 # READELF: TLS     LOCAL  DEFAULT [[#]] s_tldo_hix22
 # READELF: TLS     GLOBAL DEFAULT   UND s_tle_lox10
-# READELF: TLS     GLOBAL DEFAULT   UND s_tie_hi22
-# READELF: TLS     GLOBAL DEFAULT   UND s_tie_lo10
-# READELF: TLS     GLOBAL DEFAULT   UND s_tie_ld
-# READELF: TLS     GLOBAL DEFAULT   UND s_tie_ldx
-# READELF: TLS     GLOBAL DEFAULT   UND s_tie_add
+# READELF-V9: TLS     GLOBAL DEFAULT   UND s_tie_hi22
+# READELF-V9: TLS     GLOBAL DEFAULT   UND s_tie_lo10
+# READELF-V9: TLS     GLOBAL DEFAULT   UND s_tie_ld
+# READELF-V9: TLS     GLOBAL DEFAULT   UND s_tie_ldx
+# READELF-V9: TLS     GLOBAL DEFAULT   UND s_tie_add
 # READELF: TLS     GLOBAL DEFAULT   UND s_tldm_hi22
 # READELF: TLS     GLOBAL DEFAULT   UND s_tldm_lo10
 # READELF: TLS     GLOBAL DEFAULT   UND s_tldm_add
@@ -72,23 +73,24 @@ or %g1, %hm(sym), %g3
 or %g1, %ulo(sym), %g3
 sethi %lm(sym), %l0
 
-# ASM:      sethi %hix(sym), %g1
-# ASM-NEXT: xor %g1, %lox(sym), %g1
-# ASM-NEXT: sethi %gdop_hix22(sym), %l1
-# ASM-NEXT: or %l1, %gdop_lox10(sym), %l1
-# ASM-NEXT: ldx [%l7+%l1], %l2, %gdop(sym)
-# OBJDUMP:      sethi 0x3fffff, %g0
-# OBJDUMP-NEXT: xor %g0, -0x400, %g0
-# OBJDUMP-NEXT: sethi 0x0, %g1
-# OBJDUMP-NEXT:   R_SPARC_HIX22 sym
-# OBJDUMP-NEXT: xor %g1, 0x0, %g1
-# OBJDUMP-NEXT:   R_SPARC_LOX10 sym
-# OBJDUMP-NEXT: sethi 0x0, %l1
-# OBJDUMP-NEXT:   R_SPARC_GOTDATA_OP_HIX22 sym
-# OBJDUMP-NEXT: or %l1, 0x0, %l1
-# OBJDUMP-NEXT:   R_SPARC_GOTDATA_OP_LOX10 sym
-# OBJDUMP-NEXT: ldx [%l7+%l1], %l2
-# OBJDUMP-NEXT:   R_SPARC_GOTDATA_OP sym
+.ifdef V9
+# ASM-V9:      sethi %hix(sym), %g1
+# ASM-V9-NEXT: xor %g1, %lox(sym), %g1
+# ASM-V9-NEXT: sethi %gdop_hix22(sym), %l1
+# ASM-V9-NEXT: or %l1, %gdop_lox10(sym), %l1
+# ASM-V9-NEXT: ldx [%l7+%l1], %l2, %gdop(sym)
+# OBJDUMP-V9:      sethi 0x3fffff, %g0
+# OBJDUMP-V9-NEXT: xor %g0, -0x400, %g0
+# OBJDUMP-V9-NEXT: sethi 0x0, %g1
+# OBJDUMP-V9-NEXT:   R_SPARC_HIX22 sym
+# OBJDUMP-V9-NEXT: xor %g1, 0x0, %g1
+# OBJDUMP-V9-NEXT:   R_SPARC_LOX10 sym
+# OBJDUMP-V9-NEXT: sethi 0x0, %l1
+# OBJDUMP-V9-NEXT:   R_SPARC_GOTDATA_OP_HIX22 sym
+# OBJDUMP-V9-NEXT: or %l1, 0x0, %l1
+# OBJDUMP-V9-NEXT:   R_SPARC_GOTDATA_OP_LOX10 sym
+# OBJDUMP-V9-NEXT: ldx [%l7+%l1], %l2
+# OBJDUMP-V9-NEXT:   R_SPARC_GOTDATA_OP sym
 sethi %hix(zero), %g0
 xor %g0, %lox(zero), %g0
 sethi %hix(sym), %g1
@@ -96,6 +98,7 @@ xor %g1, %lox(sym), %g1
 sethi %gdop_hix22(sym), %l1
 or %l1, %gdop_lox10(sym), %l1
 ldx [%l7 + %l1], %l2, %gdop(sym)
+.endif
 
 .set abs, 0xfedcba98
 .set abs48, 0xfedcba987654
@@ -147,23 +150,25 @@ xor %o0, %lox(abs), %o0
         sethi %tle_hix22(s_tle_hix22), %i0
         xor %i0, %tle_lox10(s_tle_lox10), %i0
 
+.ifdef V9
 ## Initial Executable model
-# ASM:      sethi %tie_hi22(s_tie_hi22), %i1
-# ASM-NEXT: add %i1, %tie_lo10(s_tie_lo10), %i1
-# ASM-NEXT: ld [%i0+%i1], %i0, %tie_ld(s_tie_ld)
-# ASM-NEXT: ldx [%i0+%i1], %i0, %tie_ldx(s_tie_ldx)
-# ASM-NEXT: add %g7, %i0, %o0, %tie_add(s_tie_add)
-
-# OBJDUMP:      R_SPARC_TLS_IE_HI22	s_tie_hi22
-# OBJDUMP:      R_SPARC_TLS_IE_LO10	s_tie_lo10
-# OBJDUMP:      R_SPARC_TLS_IE_LD	s_tie_ld
-# OBJDUMP:      R_SPARC_TLS_IE_LDX	s_tie_ldx
-# OBJDUMP:      R_SPARC_TLS_IE_ADD	s_tie_add
+# ASM-V9:      sethi %tie_hi22(s_tie_hi22), %i1
+# ASM-V9-NEXT: add %i1, %tie_lo10(s_tie_lo10), %i1
+# ASM-V9-NEXT: ld [%i0+%i1], %i0, %tie_ld(s_tie_ld)
+# ASM-V9-NEXT: ldx [%i0+%i1], %i0, %tie_ldx(s_tie_ldx)
+# ASM-V9-NEXT: add %g7, %i0, %o0, %tie_add(s_tie_add)
+
+# OBJDUMP-V9:      R_SPARC_TLS_IE_HI22	s_tie_hi22
+# OBJDUMP-V9:      R_SPARC_TLS_IE_LO10	s_tie_lo10
+# OBJDUMP-V9:      R_SPARC_TLS_IE_LD	s_tie_ld
+# OBJDUMP-V9:      R_SPARC_TLS_IE_LDX	s_tie_ldx
+# OBJDUMP-V9:      R_SPARC_TLS_IE_ADD	s_tie_add
 	sethi %tie_hi22(s_tie_hi22), %i1
         add %i1, %tie_lo10(s_tie_lo10), %i1
         ld [%i0+%i1], %i0, %tie_ld(s_tie_ld)
         ldx [%i0+%i1], %i0, %tie_ldx(s_tie_ldx)
         add %g7, %i0, %o0, %tie_add(s_tie_add)
+.endif
 
 ## Local Dynamic model
 # ASM:      sethi %tldo_hix22(s_tldo_hix22), %i1
diff --git a/llvm/test/MC/Sparc/sparcv9-instructions.s b/llvm/test/MC/Sparc/sparcv9-instructions.s
index de9fe03201b1..6dd0dc3d64e0 100644
--- a/llvm/test/MC/Sparc/sparcv9-instructions.s
+++ b/llvm/test/MC/Sparc/sparcv9-instructions.s
@@ -53,14 +53,19 @@
         ! V9: lda [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0x86,0x10,0x76]
         lduwa [%i0 + %l6] (130+1), %o2
 
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldsw [%i0+%l6], %o2    ! encoding: [0xd4,0x46,0x00,0x16]
         ldsw [%i0 + %l6], %o2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldsw [%i0+32], %o2     ! encoding: [0xd4,0x46,0x20,0x20]
         ldsw [%i0 + 32], %o2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldsw [%g1], %o2        ! encoding: [0xd4,0x40,0x40,0x00]
         ldsw [%g1], %o2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldswa [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0xc6,0x10,0x76]
         ldswa [%i0 + %l6] 131, %o2
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldswa [%i0+%l6] #ASI_SNF, %o2 ! encoding: [0xd4,0xc6,0x10,0x76]
         ldswa [%i0 + %l6] (130+1), %o2
 
@@ -121,8 +126,10 @@
         ! V9: ldx [%g2+%i5], %fsr   ! encoding: [0xc3,0x08,0x80,0x1d]
         ldx [%g2 + %i5],%fsr
 
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldxa [%g2+%i5] #ASI_SNF, %g0   ! encoding: [0xc0,0xd8,0x90,0x7d]
         ldxa [%g2 + %i5] 131, %g0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldxa [%g2+%i5] #ASI_SNF, %g0   ! encoding: [0xc0,0xd8,0x90,0x7d]
         ldxa [%g2 + %i5] (130+1), %g0
 
@@ -134,8 +141,10 @@
         ! V9: stx %fsr, [%g2+%i5]   ! encoding: [0xc3,0x28,0x80,0x1d]
         stx %fsr,[%g2 + %i5]
 
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] 131
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] (130+1)
 
@@ -398,33 +407,45 @@
         ! V9: wr %i0, 1, %asr21         ! encoding: [0xab,0x86,0x20,0x01]
         wr %i0, 1, %clear_softint
 
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: st %o1, [%o0]             ! encoding: [0xd2,0x22,0x00,0x00]
         stw %o1, [%o0]
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: st %o1, [%o0]             ! encoding: [0xd2,0x22,0x00,0x00]
         stuw %o1, [%o0]
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: st %o1, [%o0]             ! encoding: [0xd2,0x22,0x00,0x00]
         stsw %o1, [%o0]
 
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: sta %o2, [%i0+%l6] #ASI_SNF ! encoding: [0xd4,0xa6,0x10,0x76]
         stwa %o2, [%i0 + %l6] 131
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: sta %o2, [%i0+%l6] #ASI_SNF ! encoding: [0xd4,0xa6,0x10,0x76]
         stuwa %o2, [%i0 + %l6] 131
+        ! V8:      [[#@LINE+2]]:9: error: invalid instruction mnemonic
         ! V9: sta %o2, [%i0+%l6] #ASI_SNF ! encoding: [0xd4,0xa6,0x10,0x76]
         stswa %o2, [%i0 + %l6] 131
 
         !! SPARCv9 provides a new variant of ASI-tagged memory accesses.
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldxa [%g2] %asi, %g0    ! encoding: [0xc0,0xd8,0xa0,0x00]
         ldxa [%g2] %asi, %g0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stxa %g0, [%g2] %asi    ! encoding: [0xc0,0xf0,0xa0,0x00]
         stxa %g0, [%g2] %asi
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldxa [%g2+5] %asi, %g0    ! encoding: [0xc0,0xd8,0xa0,0x05]
         ldxa [%g2 + 5] %asi, %g0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stxa %g0, [%g2+5] %asi    ! encoding: [0xc0,0xf0,0xa0,0x05]
         stxa %g0, [%g2 + 5] %asi
 
         !! Also make sure named ASI tags are parsed properly.
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: ldxa [%g2+%i5] #ASI_SNF, %g0   ! encoding: [0xc0,0xd8,0x90,0x7d]
         ldxa [%g2 + %i5] #ASI_SNF, %g0
+        ! V8:      [[#@LINE+2]]:9: error: instruction requires a CPU feature not currently enabled
         ! V9: stxa %g0, [%g2+%i5] #ASI_SNF   ! encoding: [0xc0,0xf0,0x90,0x7d]
         stxa %g0, [%g2 + %i5] #ASI_SNF
 
-- 
cgit v1.2.3


From b96370131d1572feb9c51442ac8ba1ccb16d7071 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 19 Jun 2025 15:29:56 -0700
Subject: [TTI] Plumb CostKind through getPartialReductionCost (#144953)

Purely for the sake of being idiomatic with other TTI costing routines,
no direct motivation beyond that.
---
 llvm/include/llvm/Analysis/TargetTransformInfo.h              |  4 ++--
 llvm/include/llvm/Analysis/TargetTransformInfoImpl.h          | 11 +++++------
 llvm/lib/Analysis/TargetTransformInfo.cpp                     |  5 +++--
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp        |  7 +++++--
 llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h          | 11 +++++------
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp            |  9 ++++-----
 llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h              | 11 +++++------
 .../lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp |  7 +++++--
 llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h  |  4 ++--
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp               |  2 +-
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp                |  6 +++---
 11 files changed, 40 insertions(+), 37 deletions(-)

diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 9dc4eca82492..ba47cef274be 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1332,8 +1332,8 @@ public:
   LLVM_ABI InstructionCost getPartialReductionCost(
       unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
       ElementCount VF, PartialReductionExtendKind OpAExtend,
-      PartialReductionExtendKind OpBExtend,
-      std::optional<unsigned> BinOp = std::nullopt) const;
+      PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+      TTI::TargetCostKind CostKind) const;
 
   /// \return The maximum interleave factor that any transform should try to
   /// perform for this target. This number depends on the level of parallelism
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index d93375218394..640766cf8cd1 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -652,12 +652,11 @@ public:
   virtual bool enableWritePrefetching() const { return false; }
   virtual bool shouldPrefetchAddressSpace(unsigned AS) const { return !AS; }
 
-  virtual InstructionCost
-  getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB,
-                          Type *AccumType, ElementCount VF,
-                          TTI::PartialReductionExtendKind OpAExtend,
-                          TTI::PartialReductionExtendKind OpBExtend,
-                          std::optional<unsigned> BinOp = std::nullopt) const {
+  virtual InstructionCost getPartialReductionCost(
+      unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+      ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+      TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+      TTI::TargetCostKind CostKind) const {
     return InstructionCost::getInvalid();
   }
 
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index d9cb11de9c09..8cc7f8a9d2ab 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -871,10 +871,11 @@ bool TargetTransformInfo::shouldPrefetchAddressSpace(unsigned AS) const {
 InstructionCost TargetTransformInfo::getPartialReductionCost(
     unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
     ElementCount VF, PartialReductionExtendKind OpAExtend,
-    PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp) const {
+    PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+    TTI::TargetCostKind CostKind) const {
   return TTIImpl->getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
                                           AccumType, VF, OpAExtend, OpBExtend,
-                                          BinOp);
+                                          BinOp, CostKind);
 }
 
 unsigned TargetTransformInfo::getMaxInterleaveFactor(ElementCount VF) const {
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index ed051f295752..9d5c984fa4f1 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5395,11 +5395,14 @@ AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
 InstructionCost AArch64TTIImpl::getPartialReductionCost(
     unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
     ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
-    TTI::PartialReductionExtendKind OpBExtend,
-    std::optional<unsigned> BinOp) const {
+    TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+    TTI::TargetCostKind CostKind) const {
   InstructionCost Invalid = InstructionCost::getInvalid();
   InstructionCost Cost(TTI::TCC_Basic);
 
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return Invalid;
+
   // Sub opcodes currently only occur in chained cases.
   // Independent partial reduction subtractions are still costed as an add
   if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
index 0184e748b3d8..470af01be315 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
@@ -382,12 +382,11 @@ public:
     return BaseT::isLegalNTLoad(DataType, Alignment);
   }
 
-  InstructionCost
-  getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB,
-                          Type *AccumType, ElementCount VF,
-                          TTI::PartialReductionExtendKind OpAExtend,
-                          TTI::PartialReductionExtendKind OpBExtend,
-                          std::optional<unsigned> BinOp) const override;
+  InstructionCost getPartialReductionCost(
+      unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+      ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+      TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+      TTI::TargetCostKind CostKind) const override;
 
   bool enableOrderedReductions() const override { return true; }
 
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
index 63c5f17a8487..1b80b0fcaf10 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
@@ -297,8 +297,8 @@ RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
 InstructionCost RISCVTTIImpl::getPartialReductionCost(
     unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
     ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
-    TTI::PartialReductionExtendKind OpBExtend,
-    std::optional<unsigned> BinOp) const {
+    TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+    TTI::TargetCostKind CostKind) const {
 
   // zve32x is broken for partial_reduce_umla, but let's make sure we
   // don't generate them.
@@ -311,9 +311,8 @@ InstructionCost RISCVTTIImpl::getPartialReductionCost(
   Type *Tp = VectorType::get(AccumType, VF.divideCoefficientBy(4));
   std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp);
   // Note: Asuming all vqdot* variants are equal cost
-  // TODO: Thread CostKind through this API
-  return LT.first * getRISCVInstructionCost(RISCV::VQDOT_VV, LT.second,
-                                            TTI::TCK_RecipThroughput);
+  return LT.first *
+         getRISCVInstructionCost(RISCV::VQDOT_VV, LT.second, CostKind);
 }
 
 bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
index 75d377abb0e7..83ac71ed9da6 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
+++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
@@ -100,12 +100,11 @@ public:
   TargetTransformInfo::PopcntSupportKind
   getPopcntSupport(unsigned TyWidth) const override;
 
-  InstructionCost
-  getPartialReductionCost(unsigned Opcode, Type *InputTypeA, Type *InputTypeB,
-                          Type *AccumType, ElementCount VF,
-                          TTI::PartialReductionExtendKind OpAExtend,
-                          TTI::PartialReductionExtendKind OpBExtend,
-                          std::optional<unsigned> BinOp) const override;
+  InstructionCost getPartialReductionCost(
+      unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
+      ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
+      TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+      TTI::TargetCostKind CostKind) const override;
 
   bool shouldExpandReduction(const IntrinsicInst *II) const override;
   bool supportsScalableVectors() const override {
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
index 978e08bb8955..4f159996e4c6 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp
@@ -198,12 +198,15 @@ InstructionCost WebAssemblyTTIImpl::getVectorInstrCost(
 InstructionCost WebAssemblyTTIImpl::getPartialReductionCost(
     unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
     ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
-    TTI::PartialReductionExtendKind OpBExtend,
-    std::optional<unsigned> BinOp) const {
+    TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+    TTI::TargetCostKind CostKind) const {
   InstructionCost Invalid = InstructionCost::getInvalid();
   if (!VF.isFixed() || !ST->hasSIMD128())
     return Invalid;
 
+  if (CostKind != TTI::TCK_RecipThroughput)
+    return Invalid;
+
   InstructionCost Cost(TTI::TCC_Basic);
 
   // Possible options:
diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
index 6b6d060076a8..d83b8d1f45db 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h
@@ -86,8 +86,8 @@ public:
   InstructionCost getPartialReductionCost(
       unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
       ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
-      TTI::PartialReductionExtendKind OpBExtend,
-      std::optional<unsigned> BinOp = std::nullopt) const override;
+      TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
+      TTI::TargetCostKind CostKind) const override;
   TTI::ReductionShuffle
   getPreferredExpandedReductionShuffle(const IntrinsicInst *II) const override;
 
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index e14f985efd96..9a2cd94eda58 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8240,7 +8240,7 @@ bool VPRecipeBuilder::getScaledReductions(
           [&](ElementCount VF) {
             InstructionCost Cost = TTI->getPartialReductionCost(
                 Update->getOpcode(), A->getType(), B->getType(), PHI->getType(),
-                VF, OpAExtend, OpBExtend, BinOp->getOpcode());
+                VF, OpAExtend, OpBExtend, BinOp->getOpcode(), CM.CostKind);
             return Cost.isValid();
           },
           Range)) {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f3b5c8cfa988..22861eb1c7df 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -336,9 +336,9 @@ VPPartialReductionRecipe::computeCost(ElementCount VF,
     return TargetTransformInfo::PR_None;
   };
 
-  return Ctx.TTI.getPartialReductionCost(getOpcode(), InputTypeA, InputTypeB,
-                                         PhiType, VF, GetExtendKind(ExtAR),
-                                         GetExtendKind(ExtBR), Opcode);
+  return Ctx.TTI.getPartialReductionCost(
+      getOpcode(), InputTypeA, InputTypeB, PhiType, VF, GetExtendKind(ExtAR),
+      GetExtendKind(ExtBR), Opcode, Ctx.CostKind);
 }
 
 void VPPartialReductionRecipe::execute(VPTransformState &State) {
-- 
cgit v1.2.3


From d8e6d74c6905b3032a3dc9b686bd80bb3feb9857 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Thu, 19 Jun 2025 16:15:23 -0700
Subject: [LV] Consider EVL legality for TTI tail folding preference (#144790)

---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 43 ++++++++++++-------------
 1 file changed, 20 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9a2cd94eda58..d9f53c4146c2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -1364,36 +1364,33 @@ public:
       return;
     }
 
-    if (!ForceTailFoldingStyle.getNumOccurrences()) {
-      ChosenTailFoldingStyle = {
-          TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
-          TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
-      return;
-    }
+    // Default to TTI preference, but allow command line override.
+    ChosenTailFoldingStyle = {
+        TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/true),
+        TTI.getPreferredTailFoldingStyle(/*IVUpdateMayOverflow=*/false)};
+    if (ForceTailFoldingStyle.getNumOccurrences())
+      ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
+                                ForceTailFoldingStyle.getValue()};
 
-    // Set styles when forced.
-    ChosenTailFoldingStyle = {ForceTailFoldingStyle.getValue(),
-                              ForceTailFoldingStyle.getValue()};
     if (ForceTailFoldingStyle != TailFoldingStyle::DataWithEVL)
       return;
     // Override forced styles if needed.
     // FIXME: Investigate opportunity for fixed vector factor.
     bool EVLIsLegal = UserIC <= 1 && IsScalableVF &&
                       TTI.hasActiveVectorLength() && !EnableVPlanNativePath;
-    if (!EVLIsLegal) {
-      // If for some reason EVL mode is unsupported, fallback to
-      // DataWithoutLaneMask to try to vectorize the loop with folded tail
-      // in a generic way.
-      ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
-                                TailFoldingStyle::DataWithoutLaneMask};
-      LLVM_DEBUG(
-          dbgs()
-          << "LV: Preference for VP intrinsics indicated. Will "
-             "not try to generate VP Intrinsics "
-          << (UserIC > 1
-                  ? "since interleave count specified is greater than 1.\n"
-                  : "due to non-interleaving reasons.\n"));
-    }
+    if (EVLIsLegal)
+      return;
+    // If for some reason EVL mode is unsupported, fallback to
+    // DataWithoutLaneMask to try to vectorize the loop with folded tail
+    // in a generic way.
+    ChosenTailFoldingStyle = {TailFoldingStyle::DataWithoutLaneMask,
+                              TailFoldingStyle::DataWithoutLaneMask};
+    LLVM_DEBUG(
+        dbgs() << "LV: Preference for VP intrinsics indicated. Will "
+                  "not try to generate VP Intrinsics "
+               << (UserIC > 1
+                       ? "since interleave count specified is greater than 1.\n"
+                       : "due to non-interleaving reasons.\n"));
   }
 
   /// Returns true if all loop blocks should be masked to fold tail loop.
-- 
cgit v1.2.3


From dad64877c811dcce7e2c7ebc216161ecf0733fcf Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 16:24:46 -0700
Subject: [llvm] Remove an extraneous cast (NFC) (#144955)

llvm::CallBase::getArgOperand returns Value *, so we do not need
const_cast<Value *>.
---
 llvm/include/llvm/IR/IntrinsicInst.h              | 64 +++++++++--------------
 llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp |  2 +-
 llvm/lib/IR/IntrinsicInst.cpp                     |  6 +--
 3 files changed, 29 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h
index 20e667524fa0..d8f8bdeb01d4 100644
--- a/llvm/include/llvm/IR/IntrinsicInst.h
+++ b/llvm/include/llvm/IR/IntrinsicInst.h
@@ -779,8 +779,8 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getLHS() const { return const_cast<Value *>(getArgOperand(0)); }
-  Value *getRHS() const { return const_cast<Value *>(getArgOperand(1)); }
+  Value *getLHS() const { return getArgOperand(0); }
+  Value *getRHS() const { return getArgOperand(1); }
 
   /// Returns the comparison predicate underlying the intrinsic.
   static ICmpInst::Predicate getPredicate(Intrinsic::ID ID) {
@@ -868,8 +868,8 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getLHS() const { return const_cast<Value *>(getArgOperand(0)); }
-  Value *getRHS() const { return const_cast<Value *>(getArgOperand(1)); }
+  Value *getLHS() const { return getArgOperand(0); }
+  Value *getRHS() const { return getArgOperand(1); }
 
   static bool isSigned(Intrinsic::ID ID) { return ID == Intrinsic::scmp; }
   bool isSigned() const { return isSigned(getIntrinsicID()); }
@@ -914,8 +914,8 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getLHS() const { return const_cast<Value *>(getArgOperand(0)); }
-  Value *getRHS() const { return const_cast<Value *>(getArgOperand(1)); }
+  Value *getLHS() const { return getArgOperand(0); }
+  Value *getRHS() const { return getArgOperand(1); }
 
   /// Returns the binary operation underlying the intrinsic.
   LLVM_ABI Instruction::BinaryOps getBinaryOp() const;
@@ -1118,7 +1118,7 @@ private:
 
 public:
   ConstantInt *getVolatileCst() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(ARG_VOLATILE)));
+    return cast<ConstantInt>(getArgOperand(ARG_VOLATILE));
   }
 
   bool isVolatile() const { return !getVolatileCst()->isZero(); }
@@ -1180,7 +1180,7 @@ private:
 
 public:
   ConstantInt *getVolatileCst() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(ARG_VOLATILE)));
+    return cast<ConstantInt>(getArgOperand(ARG_VOLATILE));
   }
 
   bool isVolatile() const { return !getVolatileCst()->isZero(); }
@@ -1288,7 +1288,7 @@ public:
 
   Value *getRawElementSizeInBytes() const {
     assert(isAtomic());
-    return const_cast<Value *>(getArgOperand(ARG_ELEMENTSIZE));
+    return getArgOperand(ARG_ELEMENTSIZE);
   }
 
   uint32_t getElementSizeInBytes() const {
@@ -1388,7 +1388,7 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getArgList() const { return const_cast<Value *>(getArgOperand(0)); }
+  Value *getArgList() const { return getArgOperand(0); }
 };
 
 /// This represents the llvm.va_end intrinsic.
@@ -1401,7 +1401,7 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getArgList() const { return const_cast<Value *>(getArgOperand(0)); }
+  Value *getArgList() const { return getArgOperand(0); }
 };
 
 /// This represents the llvm.va_copy intrinsic.
@@ -1414,8 +1414,8 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getDest() const { return const_cast<Value *>(getArgOperand(0)); }
-  Value *getSrc() const { return const_cast<Value *>(getArgOperand(1)); }
+  Value *getDest() const { return getArgOperand(0); }
+  Value *getSrc() const { return getArgOperand(1); }
 };
 
 /// A base class for all instrprof intrinsics.
@@ -1457,16 +1457,12 @@ public:
   // The "name" operand of the profile instrumentation instruction - this is the
   // operand that can be used to relate the instruction to the function it
   // belonged to at instrumentation time.
-  Value *getNameValue() const {
-    return const_cast<Value *>(getArgOperand(0))->stripPointerCasts();
-  }
+  Value *getNameValue() const { return getArgOperand(0)->stripPointerCasts(); }
 
   void setNameValue(Value *V) { setArgOperand(0, V); }
 
   // The hash of the CFG for the instrumented function.
-  ConstantInt *getHash() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1)));
-  }
+  ConstantInt *getHash() const { return cast<ConstantInt>(getArgOperand(1)); }
 };
 
 /// A base class for all instrprof counter intrinsics.
@@ -1563,18 +1559,14 @@ public:
     return isa<IntrinsicInst>(V) && classof(cast<IntrinsicInst>(V));
   }
 
-  Value *getTargetValue() const {
-    return cast<Value>(const_cast<Value *>(getArgOperand(2)));
-  }
+  Value *getTargetValue() const { return cast<Value>(getArgOperand(2)); }
 
   ConstantInt *getValueKind() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3)));
+    return cast<ConstantInt>(getArgOperand(3));
   }
 
   // Returns the value site index.
-  ConstantInt *getIndex() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(4)));
-  }
+  ConstantInt *getIndex() const { return cast<ConstantInt>(getArgOperand(4)); }
 };
 
 /// A base class for instrprof mcdc intrinsics that require global bitmap bytes.
@@ -1590,7 +1582,7 @@ public:
   /// \return The number of bits used for the MCDC bitmaps for the instrumented
   /// function.
   ConstantInt *getNumBitmapBits() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2)));
+    return cast<ConstantInt>(getArgOperand(2));
   }
 
   /// \return The number of bytes used for the MCDC bitmaps for the instrumented
@@ -1624,14 +1616,12 @@ public:
   /// \return The index of the TestVector Bitmap upon which this intrinsic
   /// acts.
   ConstantInt *getBitmapIndex() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2)));
+    return cast<ConstantInt>(getArgOperand(2));
   }
 
   /// \return The address of the corresponding condition bitmap containing
   /// the index of the TestVector to update within the TestVector Bitmap.
-  Value *getMCDCCondBitmapAddr() const {
-    return cast<Value>(const_cast<Value *>(getArgOperand(3)));
-  }
+  Value *getMCDCCondBitmapAddr() const { return cast<Value>(getArgOperand(3)); }
 };
 
 class PseudoProbeInst : public IntrinsicInst {
@@ -1645,20 +1635,16 @@ public:
   }
 
   ConstantInt *getFuncGuid() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(0)));
+    return cast<ConstantInt>(getArgOperand(0));
   }
 
-  ConstantInt *getIndex() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(1)));
-  }
+  ConstantInt *getIndex() const { return cast<ConstantInt>(getArgOperand(1)); }
 
   ConstantInt *getAttributes() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2)));
+    return cast<ConstantInt>(getArgOperand(2));
   }
 
-  ConstantInt *getFactor() const {
-    return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3)));
-  }
+  ConstantInt *getFactor() const { return cast<ConstantInt>(getArgOperand(3)); }
 };
 
 class NoAliasScopeDeclInst : public IntrinsicInst {
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index fbbbea6156fc..b4e64d7416d8 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -240,7 +240,7 @@ bool InlineAsmLowering::lowerInlineAsm(
 
     // Compute the value type for each operand.
     if (OpInfo.hasArg()) {
-      OpInfo.CallOperandVal = const_cast<Value *>(Call.getArgOperand(ArgNo));
+      OpInfo.CallOperandVal = Call.getArgOperand(ArgNo);
 
       if (isa<BasicBlock>(OpInfo.CallOperandVal)) {
         LLVM_DEBUG(dbgs() << "Basic block input operands not supported yet\n");
diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp
index 256bce1abe71..b1d3339c5a41 100644
--- a/llvm/lib/IR/IntrinsicInst.cpp
+++ b/llvm/lib/IR/IntrinsicInst.cpp
@@ -239,13 +239,13 @@ void DbgAssignIntrinsic::setValue(Value *V) {
 ConstantInt *InstrProfCntrInstBase::getNumCounters() const {
   if (InstrProfValueProfileInst::classof(this))
     llvm_unreachable("InstrProfValueProfileInst does not have counters!");
-  return cast<ConstantInt>(const_cast<Value *>(getArgOperand(2)));
+  return cast<ConstantInt>(getArgOperand(2));
 }
 
 ConstantInt *InstrProfCntrInstBase::getIndex() const {
   if (InstrProfValueProfileInst::classof(this))
     llvm_unreachable("Please use InstrProfValueProfileInst::getIndex()");
-  return cast<ConstantInt>(const_cast<Value *>(getArgOperand(3)));
+  return cast<ConstantInt>(getArgOperand(3));
 }
 
 void InstrProfCntrInstBase::setIndex(uint32_t Idx) {
@@ -255,7 +255,7 @@ void InstrProfCntrInstBase::setIndex(uint32_t Idx) {
 
 Value *InstrProfIncrementInst::getStep() const {
   if (InstrProfIncrementInstStep::classof(this)) {
-    return const_cast<Value *>(getArgOperand(4));
+    return getArgOperand(4);
   }
   const Module *M = getModule();
   LLVMContext &Context = M->getContext();
-- 
cgit v1.2.3


From 6001a8bb945762fd128f025bb8d7969d92096772 Mon Sep 17 00:00:00 2001
From: Tianle Liu <tianle.l.liu@intel.com>
Date: Fri, 20 Jun 2025 08:01:32 +0800
Subject: [WholeProgramDevirt] Add check for AvailableExternal and give up
 icall.branch.funnel (#143468)

When a customer class inherits from a libc++ class, and is built with
"-flto  -fwhole-program-vtables -static-libstdc++ \
-Wl,-plugin-opt=-whole-program-visibility", the libc++ class's vtable is
available_externally, meanwhile the customer class vtable is private.
And
both of them are !vcall_visibility == Linkage Unit.
In this case, icall.branch.funnel might be generated.

But the icall.branch.funnel would cause crash in LowerTypeTests because
available_externally Global_Object's GlobalTypeMember would not be
saved and finally leads to a NULL GlobalTypeMember which causes a crash.
Even saving the available_externally GO's GlobalTypeMember so that it is
not NULL to avoid the crash in LowerTypeTests, it still will crash in
SelectionDAGBuilder or Verifier, because operands linkage type
consistency
check of icall.branch.funnel can not pass.

So any one of available externally vtable would stop to generate
icall.branch.funnel.
This patch fixes FullLTO mode and split-LTO-unit ThinLTO mode.
---
 llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp     | 16 +++++++
 .../WholeProgramDevirt/availableexternal-check.ll  | 56 ++++++++++++++++++++++
 2 files changed, 72 insertions(+)
 create mode 100644 llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll

diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
index 30e1dc7167a3..aec484f8a18f 100644
--- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
+++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp
@@ -1450,6 +1450,22 @@ void DevirtModule::tryICallBranchFunnel(
   if (!HasNonDevirt)
     return;
 
+  // If any GV is AvailableExternally, not to generate branch.funnel.
+  // NOTE: It is to avoid crash in LowerTypeTest.
+  // If the branch.funnel is generated, because GV.isDeclarationForLinker(),
+  // in LowerTypeTestsModule::lower(), its GlobalTypeMember would NOT
+  // be saved in GlobalTypeMembers[&GV]. Then crash happens in
+  // buildBitSetsFromDisjointSet due to GlobalTypeMembers[&GV] is NULL.
+  // Even doing experiment to save it in GlobalTypeMembers[&GV] and
+  // making GlobalTypeMembers[&GV] be not NULL, crash could avoid from
+  // buildBitSetsFromDisjointSet. But still report_fatal_error in Verifier
+  // or SelectionDAGBuilder later, because operands linkage type consistency
+  // check of icall.branch.funnel can not pass.
+  for (auto &T : TargetsForSlot) {
+    if (T.TM->Bits->GV->hasAvailableExternallyLinkage())
+      return;
+  }
+
   FunctionType *FT =
       FunctionType::get(Type::getVoidTy(M.getContext()), {Int8PtrTy}, true);
   Function *JT;
diff --git a/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll b/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll
new file mode 100644
index 000000000000..41809b41d8c7
--- /dev/null
+++ b/llvm/test/Transforms/WholeProgramDevirt/availableexternal-check.ll
@@ -0,0 +1,56 @@
+; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility %s | FileCheck %s
+
+; This test is reduced from C++ code like this:
+; class A :public std::exception {
+; public:
+;   A() {};
+;   const char* what () const throw () {return "A";}
+; };
+; long test(std::exception *p) {
+;   const char* ch = p->what();
+;   ...;
+; }
+;
+; Build command is "clang++ -O2 -target x86_64-unknown-linux -flto=full \
+; -fwhole-program-vtables -static-libstdc++  -Wl,-plugin-opt=-whole-program-visibility"
+;
+; _ZTVSt9exception's visibility is 1 (Linkage Unit), and available_externally.
+; If any GV is available_externally, icall.branch.funnel should not be generated.
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux"
+
+@_ZTVSt9exception = available_externally constant { [5 x ptr] } { [5 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr @_ZNKSt9exception4whatEv] }, !type !0, !type !1
+@_ZTV1A.0 = constant [5 x ptr] [ptr null, ptr null, ptr null, ptr null, ptr @_ZNK1A4whatEv], !type !3, !type !4, !type !5, !type !6
+
+declare ptr @_ZNKSt9exception4whatEv()
+
+define ptr @_Z4testPSt9exception() {
+  %1 = load ptr, ptr null, align 8
+  %2 = call i1 @llvm.type.test(ptr %1, metadata !"_ZTSSt9exception")
+  tail call void @llvm.assume(i1 %2)
+  %3 = getelementptr i8, ptr %1, i64 16
+  %4 = load ptr, ptr %3, align 8
+  %5 = tail call ptr %4(ptr null)
+  ret ptr %5
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write)
+declare void @llvm.assume(i1 noundef) #0
+
+declare ptr @_ZNK1A4whatEv()
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i1 @llvm.type.test(ptr, metadata) #1
+
+; CHECK-NOT: call void (...) @llvm.icall.branch.funnel
+
+attributes #0 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) }
+attributes #1 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
+
+!0 = !{i64 16, !"_ZTSSt9exception"}
+!1 = !{i64 32, !"_ZTSMSt9exceptionKDoFPKcvE.virtual"}
+!3 = !{i32 16, !"_ZTS1A"}
+!4 = !{i32 32, !"_ZTSM1AKDoFPKcvE.virtual"}
+!5 = !{i32 16, !"_ZTSSt9exception"}
+!6 = !{i32 32, !"_ZTSMSt9exceptionKDoFPKcvE.virtual"}
-- 
cgit v1.2.3


From 91439817e8d19613ac6e25ca9abd5e7534a9d33b Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 09:34:35 +0900
Subject: ARM: Avoid using isTarget wrappers around Triple predicates (#144705)

These are module level properties, and querying them through
a function-level subtarget context is confusing. Plus we don't
need an aliased name. This doesn't avoid all the uses, just the
ones in the TargetLowering constructor.
---
 llvm/lib/Target/ARM/ARMISelLowering.cpp | 29 ++++++++++++++---------------
 llvm/lib/Target/ARM/ARMSubtarget.h      |  4 ++++
 2 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp
index 91fb7bc4578b..4567081fe78d 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.cpp
+++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp
@@ -515,7 +515,9 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setBooleanContents(ZeroOrOneBooleanContent);
   setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
 
-  if (Subtarget->isTargetMachO()) {
+  const Triple &TT = TM.getTargetTriple();
+
+  if (TT.isOSBinFormatMachO()) {
     // Uses VFP for Thumb libfuncs if available.
     if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
         Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
@@ -588,9 +590,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   }
 
   // RTLIB
-  if (TM.isAAPCS_ABI() &&
-      (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
-       Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
+  if (TM.isAAPCS_ABI() && (TT.isTargetAEABI() || TT.isTargetGNUAEABI() ||
+                           TT.isTargetMuslAEABI() || TT.isAndroid())) {
     // clang-format off
     static const struct {
       const RTLIB::Libcall Op;
@@ -712,7 +713,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   // The half <-> float conversion functions are always soft-float on
   // non-watchos platforms, but are needed for some targets which use a
   // hard-float calling convention by default.
-  if (!Subtarget->isTargetWatchABI()) {
+  if (!TT.isWatchABI()) {
     if (TM.isAAPCS_ABI()) {
       setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
       setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
@@ -726,7 +727,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
 
   // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
   // a __gnu_ prefix (which is the default).
-  if (Subtarget->isTargetAEABI()) {
+  if (TT.isTargetAEABI()) {
     static const struct {
       const RTLIB::Libcall Op;
       const char * const Name;
@@ -741,7 +742,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
       setLibcallName(LC.Op, LC.Name);
       setLibcallCallingConv(LC.Op, LC.CC);
     }
-  } else if (!Subtarget->isTargetMachO()) {
+  } else if (!TT.isOSBinFormatMachO()) {
     setLibcallName(RTLIB::FPROUND_F32_F16, "__gnu_f2h_ieee");
     setLibcallName(RTLIB::FPEXT_F16_F32, "__gnu_h2f_ieee");
   }
@@ -1227,7 +1228,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
     setOperationAction(ISD::UDIV,  MVT::i32, LibCall);
   }
 
-  if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
+  if (TT.isOSWindows() && !Subtarget->hasDivideInThumbMode()) {
     setOperationAction(ISD::SDIV, MVT::i32, Custom);
     setOperationAction(ISD::UDIV, MVT::i32, Custom);
 
@@ -1239,9 +1240,8 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setOperationAction(ISD::UREM,  MVT::i32, Expand);
 
   // Register based DivRem for AEABI (RTABI 4.2)
-  if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
-      Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
-      Subtarget->isTargetWindows()) {
+  if (TT.isTargetAEABI() || TT.isAndroid() || TT.isTargetGNUAEABI() ||
+      TT.isTargetMuslAEABI() || TT.isOSWindows()) {
     setOperationAction(ISD::SREM, MVT::i64, Custom);
     setOperationAction(ISD::UREM, MVT::i64, Custom);
     HasStandaloneRem = false;
@@ -1271,7 +1271,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   setOperationAction(ISD::STACKSAVE,          MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,       MVT::Other, Expand);
 
-  if (Subtarget->isTargetWindows())
+  if (TT.isOSWindows())
     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
   else
     setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
@@ -1326,8 +1326,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
   }
 
   // Compute supported atomic widths.
-  if (Subtarget->isTargetLinux() ||
-      (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
+  if (TT.isOSLinux() || (!Subtarget->isMClass() && Subtarget->hasV6Ops())) {
     // For targets where __sync_* routines are reliably available, we use them
     // if necessary.
     //
@@ -1538,7 +1537,7 @@ ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM_,
 
   // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined.  MinGW has
   // it, but it's just a wrapper around ldexp.
-  if (Subtarget->isTargetWindows()) {
+  if (TT.isOSWindows()) {
     for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
       if (isOperationExpand(Op, MVT::f32))
         setOperationAction(Op, MVT::f32, Promote);
diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h
index 7893796e313b..3e1314349564 100644
--- a/llvm/lib/Target/ARM/ARMSubtarget.h
+++ b/llvm/lib/Target/ARM/ARMSubtarget.h
@@ -334,6 +334,9 @@ public:
 
   const Triple &getTargetTriple() const { return TargetTriple; }
 
+  /// @{
+  /// These properties are per-module, please use the TargetMachine
+  /// TargetTriple.
   bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
   bool isTargetIOS() const { return TargetTriple.isiOS(); }
   bool isTargetWatchOS() const { return TargetTriple.isWatchOS(); }
@@ -359,6 +362,7 @@ public:
   bool isTargetEHABICompatible() const {
     return TargetTriple.isTargetEHABICompatible();
   }
+  /// @}
 
   bool isReadTPSoft() const {
     return !(isReadTPTPIDRURW() || isReadTPTPIDRURO() || isReadTPTPIDRPRW());
-- 
cgit v1.2.3


From efd42b9b1d655a56abb3e6ce1ed4414e9f882912 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 09:35:02 +0900
Subject: WebAssembly: Stop directly using RuntimeLibcalls.def (#143054)

Construct RuntimeLibcallsInfo instead of manually creating a map.
This was repeating the setting of the RETURN_ADDRESS. This removes
an obstacle to generating libcall information with tablegen.

This is also not great, since it's setting a static map which
would be broken if there were ever a triple with a different libcall
configuration.
---
 .../WebAssemblyRuntimeLibcallSignatures.cpp        | 28 +++++++++-------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
index d5c4532824c0..4548a7520b3b 100644
--- a/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
+++ b/llvm/lib/Target/WebAssembly/WebAssemblyRuntimeLibcallSignatures.cpp
@@ -528,25 +528,19 @@ RuntimeLibcallSignatureTable &getRuntimeLibcallSignatures() {
 // constructor for use with a static variable
 struct StaticLibcallNameMap {
   StringMap<RTLIB::Libcall> Map;
-  StaticLibcallNameMap() {
-    static const std::pair<const char *, RTLIB::Libcall> NameLibcalls[] = {
-#define HANDLE_LIBCALL(code, name) {(const char *)name, RTLIB::code},
-#define LIBCALL_NO_NAME nullptr
-#include "llvm/IR/RuntimeLibcalls.def"
-#undef HANDLE_LIBCALL
-#undef LIBCALL_NO_NAME
-    };
-    for (const auto &NameLibcall : NameLibcalls) {
-      if (NameLibcall.first != nullptr &&
-          getRuntimeLibcallSignatures().Table[NameLibcall.second] !=
-              unsupported) {
-        assert(!Map.contains(NameLibcall.first) &&
+  StaticLibcallNameMap(const Triple &TT) {
+    // FIXME: This is broken if there are ever different triples compiled with
+    // different libcalls.
+    RTLIB::RuntimeLibcallsInfo RTCI(TT);
+    for (RTLIB::Libcall LC : RTLIB::libcalls()) {
+      const char *NameLibcall = RTCI.getLibcallName(LC);
+      if (NameLibcall != nullptr &&
+          getRuntimeLibcallSignatures().Table[LC] != unsupported) {
+        assert(!Map.contains(NameLibcall) &&
                "duplicate libcall names in name map");
-        Map[NameLibcall.first] = NameLibcall.second;
+        Map[NameLibcall] = LC;
       }
     }
-
-    Map["emscripten_return_address"] = RTLIB::RETURN_ADDRESS;
   }
 };
 
@@ -942,7 +936,7 @@ void WebAssembly::getLibcallSignature(const WebAssemblySubtarget &Subtarget,
                                       StringRef Name,
                                       SmallVectorImpl<wasm::ValType> &Rets,
                                       SmallVectorImpl<wasm::ValType> &Params) {
-  static StaticLibcallNameMap LibcallNameMap;
+  static StaticLibcallNameMap LibcallNameMap(Subtarget.getTargetTriple());
   auto &Map = LibcallNameMap.Map;
   auto Val = Map.find(Name);
 #ifndef NDEBUG
-- 
cgit v1.2.3


From fa7646008ef32b38357189ed5752e1a1b8d6d146 Mon Sep 17 00:00:00 2001
From: joaosaffran <126493771+joaosaffran@users.noreply.github.com>
Date: Thu, 19 Jun 2025 17:39:48 -0700
Subject: [DirectX] Add Root Signature Version Support and Update Test IR
 Format (#144957)

Updates the Root Signature metadata parser to extract version
information. This requirement was added after the initial parser
implementation.

---------

Co-authored-by: joaosaffran <joao.saffran@microsoft.com>
---
 llvm/lib/Target/DirectX/DXILRootSignature.cpp      | 10 +++++--
 .../RootSignature-Error-is-not-function.ll         |  4 +--
 .../RootSignature-Error-is-not-value.ll            |  4 +--
 .../RootSignature-Error-no-root-element-list.ll    |  4 +--
 .../RootSignature-Error-root-element-not-mdnode.ll |  4 +--
 .../ContainerData/RootSignature-Flags-Error.ll     |  2 +-
 .../DirectX/ContainerData/RootSignature-Flags.ll   |  2 +-
 .../RootSignature-MultipleEntryFunctions.ll        |  4 +--
 .../RootSignature-NullFunction-Error.ll            |  4 +--
 ...ture-Parameters-Invalid-ParameterIsNotString.ll |  2 +-
 .../RootSignature-Parameters-Validation-Error.ll   |  2 +-
 .../ContainerData/RootSignature-Parameters.ll      |  2 +-
 ...gnature-RootConstants-Invalid-Num32BitValues.ll |  2 +-
 ...ignature-RootConstants-Invalid-RegisterSpace.ll |  2 +-
 ...gnature-RootConstants-Invalid-ShaderRegister.ll |  2 +-
 .../ContainerData/RootSignature-RootConstants.ll   |  2 +-
 .../RootSignature-RootDescriptor-Invalid-Flags.ll  |  2 +-
 ...ignature-RootDescriptor-Invalid-RegisterKind.ll |  2 +-
 ...gnature-RootDescriptor-Invalid-RegisterSpace.ll |  2 +-
 ...gnature-RootDescriptor-Invalid-RegisterValue.ll |  2 +-
 .../ContainerData/RootSignature-RootDescriptor.ll  |  2 +-
 .../RootSignature-RootDescriptor_V1.ll             | 34 ++++++++++++++++++++++
 .../RootSignature-RootElement-Error.ll             |  2 +-
 ...ignature-RootFlags-VisibilityValidationError.ll |  2 +-
 24 files changed, 70 insertions(+), 30 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor_V1.ll

diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 3d195acb19e1..88914a31f46e 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -309,7 +309,7 @@ analyzeModule(Module &M) {
     return RSDMap;
 
   for (const auto &RSDefNode : RootSignatureNode->operands()) {
-    if (RSDefNode->getNumOperands() != 2) {
+    if (RSDefNode->getNumOperands() != 3) {
       reportError(Ctx, "Invalid format for Root Signature Definition. Pairs "
                        "of function, root signature expected.");
       continue;
@@ -348,8 +348,14 @@ analyzeModule(Module &M) {
       reportError(Ctx, "Root Element is not a metadata node.");
       continue;
     }
-
     mcdxbc::RootSignatureDesc RSD;
+    if (std::optional<uint32_t> Version = extractMdIntValue(RSDefNode, 2))
+      RSD.Version = *Version;
+    else {
+      reportError(Ctx, "Invalid RSDefNode value, expected constant int");
+      continue;
+    }
+
     // Clang emits the root signature data in dxcontainer following a specific
     // sequence. First the header, then the root parameters. So the header
     // offset will always equal to the header size.
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-function.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-function.ll
index ad2aa7997eba..fbda7561ceca 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-function.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-function.ll
@@ -18,9 +18,9 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ i32 -1, !6 } ; function, root signature
+!5 = !{ i32 -1, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-value.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-value.ll
index 4d881f96e4c3..94ab52e1f29c 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-value.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-is-not-value.ll
@@ -18,9 +18,9 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ !3, !6 } ; function, root signature
+!5 = !{ !3, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-no-root-element-list.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-no-root-element-list.ll
index b5109022b4b0..dc7a3fd10320 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-no-root-element-list.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-no-root-element-list.ll
@@ -18,9 +18,9 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, null } ; function, root signature
+!2 = !{ ptr @main, null, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ i32 -1, !6 } ; function, root signature
+!5 = !{ i32 -1, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-root-element-not-mdnode.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-root-element-not-mdnode.ll
index 7e6bcdadd386..3028ca99e4ef 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-root-element-not-mdnode.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error-root-element-not-mdnode.ll
@@ -18,9 +18,9 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, i32 -1 } ; function, root signature
+!2 = !{ ptr @main, i32 -1, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ i32 -1, !6 } ; function, root signature
+!5 = !{ i32 -1, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags-Error.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags-Error.ll
index 4921472d253a..65511160f230 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags-Error.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags-Error.ll
@@ -15,6 +15,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"NOTRootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
index e81679732a5d..10235b7d1796 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Flags.ll
@@ -13,7 +13,7 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
 
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
index d23e1c71d2fc..fec9c226d8bc 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-MultipleEntryFunctions.ll
@@ -16,10 +16,10 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ ptr @anotherMain, !6 } ; function, root signature
+!5 = !{ ptr @anotherMain, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
 
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-NullFunction-Error.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-NullFunction-Error.ll
index f5caa5012478..c6b57ee31c87 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-NullFunction-Error.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-NullFunction-Error.ll
@@ -13,9 +13,9 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2, !5} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
-!5 = !{ null, !6 } ; function, root signature
+!5 = !{ null, !6, i32 2 } ; function, root signature
 !6 = !{ !7 } ; list of root signature elements
 !7 = !{ !"RootFlags", i32 2 } ; 1 = allow_input_assembler_input_layout
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll
index 04edd00eee64..b4b616f8fd6e 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Invalid-ParameterIsNotString.ll
@@ -14,6 +14,6 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!0}
-!0 = !{ ptr @main, !1 }
+!0 = !{ ptr @main, !1, i32 2 }
 !1 = !{ !2 }
 !2 = !{ i32 0 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Validation-Error.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Validation-Error.ll
index 2b4a075281f8..a61928d0a7fd 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Validation-Error.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters-Validation-Error.ll
@@ -15,6 +15,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootConstants", i32 255, i32 1, i32 2, i32 3 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
index 714c76213e1b..80aa757d7e10 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
@@ -11,7 +11,7 @@ entry:
 attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4, !5, !6 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
 !5 = !{ !"RootConstants", i32 0, i32 1, i32 2, i32 3 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-Num32BitValues.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-Num32BitValues.ll
index 552c128e5ab5..121bc6e932a4 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-Num32BitValues.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-Num32BitValues.ll
@@ -11,6 +11,6 @@ entry:
 }
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootConstants", i32 0, i32 1, i32 2, !"Invalid" }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-RegisterSpace.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-RegisterSpace.ll
index 1087b414942e..3534e5d1c5a2 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-RegisterSpace.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-RegisterSpace.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootConstants", i32 0, i32 1, !"Invalid", i32 3 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-ShaderRegister.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-ShaderRegister.ll
index 53fd924e8f46..5c3dce2f419e 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-ShaderRegister.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants-Invalid-ShaderRegister.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootConstants", i32 0, !"Invalid", i32 2, i32 3 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants.ll
index 71511ff52334..964554fe143e 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootConstants.ll
@@ -13,7 +13,7 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootConstants", i32 0, i32 1, i32 2, i32 3 }
 
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
index 422998124091..6c90bcb09b64 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootCBV", i32 0, i32 1, i32 2, i32 3  }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll
index 4aed84efbe2b..579528d8b5e1 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterKind.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"Invalid", i32 0, i32 1, i32 2, i32 3  }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll
index 020d117ba45d..18582090e761 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterSpace.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootCBV", i32 0, i32 1, i32 4294967280, i32 0  }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll
index edb8b943c6e3..8bbfdf00bea2 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-RegisterValue.ll
@@ -13,6 +13,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootCBV", i32 0, i32 4294967295, i32 2, i32 3  }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll
index 9217945855cd..f77bb96840be 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor.ll
@@ -13,7 +13,7 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !5 } ; list of root signature elements
 !5 = !{ !"RootCBV", i32 0, i32 1, i32 2, i32 8  }
 
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor_V1.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor_V1.ll
new file mode 100644
index 000000000000..e05c42a22ea4
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor_V1.ll
@@ -0,0 +1,34 @@
+; RUN: opt %s -dxil-embed -dxil-globals -S -o - | FileCheck %s
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: @dx.rts0 = private constant [44 x i8]  c"{{.*}}", section "RTS0", align 4
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 1 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"RootCBV", i32 0, i32 1, i32 2, i32 8  }
+
+; DXC:  - Name:            RTS0
+; DXC-NEXT:    Size:            44
+; DXC-NEXT:    RootSignature:
+; DXC-NEXT:      Version:         1
+; DXC-NEXT:      NumRootParameters: 1 
+; DXC-NEXT:      RootParametersOffset: 24 
+; DXC-NEXT:      NumStaticSamplers: 0
+; DXC-NEXT:      StaticSamplersOffset: 0
+; DXC-NEXT:      Parameters:
+; DXC-NEXT:        - ParameterType:   2
+; DXC-NEXT:          ShaderVisibility: 0
+; DXC-NEXT:          Descriptor:
+; DXC-NEXT:            RegisterSpace: 2
+; DXC-NEXT:            ShaderRegister: 1
+; DXC-NOT:            DATA_STATIC: true
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootElement-Error.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootElement-Error.ll
index 89e23f6540c5..aa8d46dccbac 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootElement-Error.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootElement-Error.ll
@@ -15,5 +15,5 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !"NOTRootElements" } ; list of root signature elements
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootFlags-VisibilityValidationError.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootFlags-VisibilityValidationError.ll
index 4b8e6abacd7a..baeb74c2c3ce 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootFlags-VisibilityValidationError.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootFlags-VisibilityValidationError.ll
@@ -15,6 +15,6 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
-!2 = !{ ptr @main, !3 } ; function, root signature
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
 !3 = !{ !4 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 2147487744 } ; 1 = allow_input_assembler_input_layout
-- 
cgit v1.2.3


From 5cbed34404a3862c2d7f18e4b4b24f5ce1516a8d Mon Sep 17 00:00:00 2001
From: Phoebe Wang <phoebe.wang@intel.com>
Date: Fri, 20 Jun 2025 08:57:37 +0800
Subject: [X86] Remove CLDEMOTE from Arrowlake and later hybrid processors
 (#144833)

Decouple Arrowlake from Sierraforest because the later has CLDEMOTE
feature.
---
 clang/test/Preprocessor/predefined-arch-macros.c | 312 ++++++++++++-----------
 llvm/lib/Target/X86/X86.td                       |  22 +-
 llvm/lib/TargetParser/X86TargetParser.cpp        |  16 +-
 3 files changed, 182 insertions(+), 168 deletions(-)

diff --git a/clang/test/Preprocessor/predefined-arch-macros.c b/clang/test/Preprocessor/predefined-arch-macros.c
index 9dfeddbd4d5a..86d51820ae5b 100644
--- a/clang/test/Preprocessor/predefined-arch-macros.c
+++ b/clang/test/Preprocessor/predefined-arch-macros.c
@@ -2556,209 +2556,211 @@
 
 // RUN: %clang -march=sierraforest -m32 -E -dM %s -o - 2>&1 \
 // RUN:     --target=i386 \
-// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SRF_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M32,CHECK_SRF_M32
 // RUN: %clang -march=grandridge -m32 -E -dM %s -o - 2>&1 \
 // RUN:     --target=i386 \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M32,CHECK_SRF_M32
 // RUN: %clang -march=arrowlake -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SRF_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ARL_M32
 // RUN: %clang -march=arrowlake-s -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M32,CHECK_ARLS_M32
 // RUN: %clang -march=lunarlake -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M32,CHECK_ARLS_M32
 // RUN: %clang -march=pantherlake -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32,CHECK_PTL_M32
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M32,CHECK_ARLS_M32,CHECK_PTL_M32
 // RUN: %clang -march=clearwaterforest -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
 // RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M32,CHECK_ARLS_M32,CHECK_PTL_M32,CHECK_CWF_M32
-// CHECK_SRF_M32: #define __ADX__ 1
-// CHECK_SRF_M32: #define __AES__ 1
-// CHECK_SRF_M32: #define __AVX2__ 1
-// CHECK_SRF_M32-NOT: AVX512
-// CHECK_SRF_M32: #define __AVXIFMA__ 1
-// CHECK_SRF_M32: #define __AVXNECONVERT__ 1
-// CHECK_SRF_M32-NOT: #define __AVXVNNIINT16__ 1
+// CHECK_ARL_M32: #define __ADX__ 1
+// CHECK_ARL_M32: #define __AES__ 1
+// CHECK_ARL_M32: #define __AVX2__ 1
+// CHECK_ARL_M32-NOT: AVX512
+// CHECK_ARL_M32: #define __AVXIFMA__ 1
+// CHECK_ARL_M32: #define __AVXNECONVERT__ 1
+// CHECK_ARL_M32-NOT: #define __AVXVNNIINT16__ 1
 // CHECK_ARLS_M32: #define __AVXVNNIINT16__ 1
-// CHECK_SRF_M32: #define __AVXVNNIINT8__ 1
-// CHECK_SRF_M32: #define __AVXVNNI__ 1
-// CHECK_SRF_M32: #define __AVX__ 1
-// CHECK_SRF_M32: #define __BMI2__ 1
-// CHECK_SRF_M32: #define __BMI__ 1
+// CHECK_ARL_M32: #define __AVXVNNIINT8__ 1
+// CHECK_ARL_M32: #define __AVXVNNI__ 1
+// CHECK_ARL_M32: #define __AVX__ 1
+// CHECK_ARL_M32: #define __BMI2__ 1
+// CHECK_ARL_M32: #define __BMI__ 1
+// CHECK_ARLS_M32-NOT: __CLDEMOTE__
 // CHECK_SRF_M32: #define __CLDEMOTE__ 1
-// CHECK_SRF_M32: #define __CLFLUSHOPT__ 1
-// CHECK_SRF_M32: #define __CLWB__ 1
-// CHECK_SRF_M32: #define __CMPCCXADD__ 1
-// CHECK_SRF_M32: #define __ENQCMD__ 1
-// CHECK_SRF_M32: #define __F16C__ 1
-// CHECK_SRF_M32: #define __FMA__ 1
-// CHECK_SRF_M32: #define __FSGSBASE__ 1
-// CHECK_SRF_M32: #define __FXSR__ 1
-// CHECK_SRF_M32: #define __GFNI__ 1
-// CHECK_SRF_M32: #define __HRESET__ 1
-// CHECK_SRF_M32: #define __INVPCID__ 1
-// CHECK_SRF_M32: #define __KL__ 1
-// CHECK_SRF_M32: #define __LZCNT__ 1
-// CHECK_SRF_M32: #define __MMX__ 1
-// CHECK_SRF_M32: #define __MOVBE__ 1
-// CHECK_SRF_M32: #define __MOVDIR64B__ 1
-// CHECK_SRF_M32: #define __MOVDIRI__ 1
-// CHECK_SRF_M32: #define __PCLMUL__ 1
-// CHECK_SRF_M32: #define __PCONFIG__ 1
-// CHECK_SRF_M32: #define __PKU__ 1
-// CHECK_SRF_M32: #define __POPCNT__ 1
-// CHECK_SRF_M32-NOT: #define __PREFETCHI__ 1
+// CHECK_ARL_M32: #define __CLFLUSHOPT__ 1
+// CHECK_ARL_M32: #define __CLWB__ 1
+// CHECK_ARL_M32: #define __CMPCCXADD__ 1
+// CHECK_ARL_M32: #define __ENQCMD__ 1
+// CHECK_ARL_M32: #define __F16C__ 1
+// CHECK_ARL_M32: #define __FMA__ 1
+// CHECK_ARL_M32: #define __FSGSBASE__ 1
+// CHECK_ARL_M32: #define __FXSR__ 1
+// CHECK_ARL_M32: #define __GFNI__ 1
+// CHECK_ARL_M32: #define __HRESET__ 1
+// CHECK_ARL_M32: #define __INVPCID__ 1
+// CHECK_ARL_M32: #define __KL__ 1
+// CHECK_ARL_M32: #define __LZCNT__ 1
+// CHECK_ARL_M32: #define __MMX__ 1
+// CHECK_ARL_M32: #define __MOVBE__ 1
+// CHECK_ARL_M32: #define __MOVDIR64B__ 1
+// CHECK_ARL_M32: #define __MOVDIRI__ 1
+// CHECK_ARL_M32: #define __PCLMUL__ 1
+// CHECK_ARL_M32: #define __PCONFIG__ 1
+// CHECK_ARL_M32: #define __PKU__ 1
+// CHECK_ARL_M32: #define __POPCNT__ 1
+// CHECK_ARL_M32-NOT: #define __PREFETCHI__ 1
 // CHECK_ARLS_M32-NOT: #define __PREFETCHI__ 1
 // CHECK_PTL_M32: #define __PREFETCHI__ 1
-// CHECK_SRF_M32: #define __PRFCHW__ 1
-// CHECK_SRF_M32: #define __PTWRITE__ 1
-// CHECK_SRF_M32-NOT: #define __RAOINT__ 1
-// CHECK_SRF_M32: #define __RDPID__ 1
-// CHECK_SRF_M32: #define __RDRND__ 1
-// CHECK_SRF_M32: #define __RDSEED__ 1
-// CHECK_SRF_M32: #define __SERIALIZE__ 1
-// CHECK_SRF_M32: #define __SGX__ 1
-// CHECK_SRF_M32-NOT: #define __SHA512__ 1
+// CHECK_ARL_M32: #define __PRFCHW__ 1
+// CHECK_ARL_M32: #define __PTWRITE__ 1
+// CHECK_ARL_M32-NOT: #define __RAOINT__ 1
+// CHECK_ARL_M32: #define __RDPID__ 1
+// CHECK_ARL_M32: #define __RDRND__ 1
+// CHECK_ARL_M32: #define __RDSEED__ 1
+// CHECK_ARL_M32: #define __SERIALIZE__ 1
+// CHECK_ARL_M32: #define __SGX__ 1
+// CHECK_ARL_M32-NOT: #define __SHA512__ 1
 // CHECK_ARLS_M32: #define __SHA512__ 1
-// CHECK_SRF_M32: #define __SHA__ 1
-// CHECK_SRF_M32: #define __SHSTK__ 1
-// CHECK_SRF_M32-NOT: #define __SM3__ 1
+// CHECK_ARL_M32: #define __SHA__ 1
+// CHECK_ARL_M32: #define __SHSTK__ 1
+// CHECK_ARL_M32-NOT: #define __SM3__ 1
 // CHECK_ARLS_M32: #define __SM3__ 1
-// CHECK_SRF_M32-NOT: #define __SM4__ 1
+// CHECK_ARL_M32-NOT: #define __SM4__ 1
 // CHECK_ARLS_M32: #define __SM4__ 1
-// CHECK_SRF_M32: #define __SSE2__ 1
-// CHECK_SRF_M32: #define __SSE3__ 1
-// CHECK_SRF_M32: #define __SSE4_1__ 1
-// CHECK_SRF_M32: #define __SSE4_2__ 1
-// CHECK_SRF_M32: #define __SSE_MATH__ 1
-// CHECK_SRF_M32: #define __SSE__ 1
-// CHECK_SRF_M32: #define __SSSE3__ 1
-// CHECK_SRF_M32: #define __UINTR__ 1
-// CHECK_SRF_M32-NOT: #define __USERMSR__ 1
+// CHECK_ARL_M32: #define __SSE2__ 1
+// CHECK_ARL_M32: #define __SSE3__ 1
+// CHECK_ARL_M32: #define __SSE4_1__ 1
+// CHECK_ARL_M32: #define __SSE4_2__ 1
+// CHECK_ARL_M32: #define __SSE_MATH__ 1
+// CHECK_ARL_M32: #define __SSE__ 1
+// CHECK_ARL_M32: #define __SSSE3__ 1
+// CHECK_ARL_M32: #define __UINTR__ 1
+// CHECK_ARL_M32-NOT: #define __USERMSR__ 1
 // CHECK_ARLS_M32-NOT: #define __USERMSR__ 1
 // CHECK_PTL_M32-NOT: #define __USERMSR__ 1
 // CHECK_CWF_M32: #define __USERMSR__ 1
-// CHECK_SRF_M32: #define __VAES__ 1
-// CHECK_SRF_M32: #define __VPCLMULQDQ__ 1
-// CHECK_SRF_M32: #define __WAITPKG__ 1
-// CHECK_SRF_M32: #define __WIDEKL__ 1
-// CHECK_SRF_M32: #define __XSAVEC__ 1
-// CHECK_SRF_M32: #define __XSAVEOPT__ 1
-// CHECK_SRF_M32: #define __XSAVES__ 1
-// CHECK_SRF_M32: #define __XSAVE__ 1
-// CHECK_SRF_M32: #define __corei7 1
-// CHECK_SRF_M32: #define __corei7__ 1
-// CHECK_SRF_M32: #define __i386 1
-// CHECK_SRF_M32: #define __i386__ 1
-// CHECK_SRF_M32: #define __tune_corei7__ 1
-// CHECK_SRF_M32: #define i386 1
+// CHECK_ARL_M32: #define __VAES__ 1
+// CHECK_ARL_M32: #define __VPCLMULQDQ__ 1
+// CHECK_ARL_M32: #define __WAITPKG__ 1
+// CHECK_ARL_M32: #define __WIDEKL__ 1
+// CHECK_ARL_M32: #define __XSAVEC__ 1
+// CHECK_ARL_M32: #define __XSAVEOPT__ 1
+// CHECK_ARL_M32: #define __XSAVES__ 1
+// CHECK_ARL_M32: #define __XSAVE__ 1
+// CHECK_ARL_M32: #define __corei7 1
+// CHECK_ARL_M32: #define __corei7__ 1
+// CHECK_ARL_M32: #define __i386 1
+// CHECK_ARL_M32: #define __i386__ 1
+// CHECK_ARL_M32: #define __tune_corei7__ 1
+// CHECK_ARL_M32: #define i386 1
 
 // RUN: %clang -march=sierraforest -m64 -E -dM %s -o - 2>&1 \
 // RUN:     --target=i386 \
-// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SRF_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_SRF_M64
 // RUN: %clang -march=grandridge -m64 -E -dM %s -o - 2>&1 \
 // RUN:     --target=i386 \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_SRF_M64
 // RUN: %clang -march=arrowlake -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_SRF_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefix=CHECK_ARL_M64
 // RUN: %clang -march=arrowlake-s -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_ARLS_M64
 // RUN: %clang -march=lunarlake -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_ARLS_M64
 // RUN: %clang -march=pantherlake -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64,CHECK_PTL_M64
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_ARLS_M64,CHECK_PTL_M64
 // RUN: %clang -march=clearwaterforest -m64 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
-// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_SRF_M64,CHECK_ARLS_M64,CHECK_PTL_M64,CHECK_CWF_M64
-// CHECK_SRF_M64: #define __ADX__ 1
-// CHECK_SRF_M64: #define __AES__ 1
-// CHECK_SRF_M64: #define __AVX2__ 1
-// CHECK_SRF_M64-NOT: AVX512
-// CHECK_SRF_M64: #define __AVXIFMA__ 1
-// CHECK_SRF_M64: #define __AVXNECONVERT__ 1
-// CHECK_SRF_M64-NOT: #define __AVXVNNIINT16__ 1
+// RUN:   | FileCheck -match-full-lines %s -check-prefixes=CHECK_ARL_M64,CHECK_SRF_M64,CHECK_ARLS_M64,CHECK_PTL_M64,CHECK_CWF_M64
+// CHECK_ARL_M64: #define __ADX__ 1
+// CHECK_ARL_M64: #define __AES__ 1
+// CHECK_ARL_M64: #define __AVX2__ 1
+// CHECK_ARL_M64-NOT: AVX512
+// CHECK_ARL_M64: #define __AVXIFMA__ 1
+// CHECK_ARL_M64: #define __AVXNECONVERT__ 1
+// CHECK_ARL_M64-NOT: #define __AVXVNNIINT16__ 1
 // CHECK_ARLS_M64: #define __AVXVNNIINT16__ 1
-// CHECK_SRF_M64: #define __AVXVNNIINT8__ 1
-// CHECK_SRF_M64: #define __AVXVNNI__ 1
-// CHECK_SRF_M64: #define __AVX__ 1
-// CHECK_SRF_M64: #define __BMI2__ 1
-// CHECK_SRF_M64: #define __BMI__ 1
+// CHECK_ARL_M64: #define __AVXVNNIINT8__ 1
+// CHECK_ARL_M64: #define __AVXVNNI__ 1
+// CHECK_ARL_M64: #define __AVX__ 1
+// CHECK_ARL_M64: #define __BMI2__ 1
+// CHECK_ARL_M64: #define __BMI__ 1
+// CHECK_ARLS_M64-NOT: __CLDEMOTE__
 // CHECK_SRF_M64: #define __CLDEMOTE__ 1
-// CHECK_SRF_M64: #define __CLFLUSHOPT__ 1
-// CHECK_SRF_M64: #define __CLWB__ 1
-// CHECK_SRF_M64: #define __CMPCCXADD__ 1
-// CHECK_SRF_M64: #define __ENQCMD__ 1
-// CHECK_SRF_M64: #define __F16C__ 1
-// CHECK_SRF_M64: #define __FMA__ 1
-// CHECK_SRF_M64: #define __FSGSBASE__ 1
-// CHECK_SRF_M64: #define __FXSR__ 1
-// CHECK_SRF_M64: #define __GFNI__ 1
-// CHECK_SRF_M64: #define __HRESET__ 1
-// CHECK_SRF_M64: #define __INVPCID__ 1
-// CHECK_SRF_M64: #define __KL__ 1
-// CHECK_SRF_M64: #define __LZCNT__ 1
-// CHECK_SRF_M64: #define __MMX__ 1
-// CHECK_SRF_M64: #define __MOVBE__ 1
-// CHECK_SRF_M64: #define __MOVDIR64B__ 1
-// CHECK_SRF_M64: #define __MOVDIRI__ 1
-// CHECK_SRF_M64: #define __PCLMUL__ 1
-// CHECK_SRF_M64: #define __PCONFIG__ 1
-// CHECK_SRF_M64: #define __PKU__ 1
-// CHECK_SRF_M64: #define __POPCNT__ 1
-// CHECK_SRF_M64-NOT: #define __PREFETCHI__ 1
+// CHECK_ARL_M64: #define __CLFLUSHOPT__ 1
+// CHECK_ARL_M64: #define __CLWB__ 1
+// CHECK_ARL_M64: #define __CMPCCXADD__ 1
+// CHECK_ARL_M64: #define __ENQCMD__ 1
+// CHECK_ARL_M64: #define __F16C__ 1
+// CHECK_ARL_M64: #define __FMA__ 1
+// CHECK_ARL_M64: #define __FSGSBASE__ 1
+// CHECK_ARL_M64: #define __FXSR__ 1
+// CHECK_ARL_M64: #define __GFNI__ 1
+// CHECK_ARL_M64: #define __HRESET__ 1
+// CHECK_ARL_M64: #define __INVPCID__ 1
+// CHECK_ARL_M64: #define __KL__ 1
+// CHECK_ARL_M64: #define __LZCNT__ 1
+// CHECK_ARL_M64: #define __MMX__ 1
+// CHECK_ARL_M64: #define __MOVBE__ 1
+// CHECK_ARL_M64: #define __MOVDIR64B__ 1
+// CHECK_ARL_M64: #define __MOVDIRI__ 1
+// CHECK_ARL_M64: #define __PCLMUL__ 1
+// CHECK_ARL_M64: #define __PCONFIG__ 1
+// CHECK_ARL_M64: #define __PKU__ 1
+// CHECK_ARL_M64: #define __POPCNT__ 1
+// CHECK_ARL_M64-NOT: #define __PREFETCHI__ 1
 // CHECK_ARLS_M64-NOT: #define __PREFETCHI__ 1
 // CHECK_PTL_M64: #define __PREFETCHI__ 1
-// CHECK_SRF_M64: #define __PRFCHW__ 1
-// CHECK_SRF_M64: #define __PTWRITE__ 1
-// CHECK_SRF_M64-NOT: #define __RAOINT__ 1
-// CHECK_SRF_M64: #define __RDPID__ 1
-// CHECK_SRF_M64: #define __RDRND__ 1
-// CHECK_SRF_M64: #define __RDSEED__ 1
-// CHECK_SRF_M64: #define __SERIALIZE__ 1
-// CHECK_SRF_M64: #define __SGX__ 1
-// CHECK_SRF_M64-NOT: #define __SHA512__ 1
+// CHECK_ARL_M64: #define __PRFCHW__ 1
+// CHECK_ARL_M64: #define __PTWRITE__ 1
+// CHECK_ARL_M64-NOT: #define __RAOINT__ 1
+// CHECK_ARL_M64: #define __RDPID__ 1
+// CHECK_ARL_M64: #define __RDRND__ 1
+// CHECK_ARL_M64: #define __RDSEED__ 1
+// CHECK_ARL_M64: #define __SERIALIZE__ 1
+// CHECK_ARL_M64: #define __SGX__ 1
+// CHECK_ARL_M64-NOT: #define __SHA512__ 1
 // CHECK_ARLS_M64: #define __SHA512__ 1
-// CHECK_SRF_M64: #define __SHA__ 1
-// CHECK_SRF_M64: #define __SHSTK__ 1
-// CHECK_SRF_M64-NOT: #define __SM3__ 1
+// CHECK_ARL_M64: #define __SHA__ 1
+// CHECK_ARL_M64: #define __SHSTK__ 1
+// CHECK_ARL_M64-NOT: #define __SM3__ 1
 // CHECK_ARLS_M64: #define __SM3__ 1
-// CHECK_SRF_M64-NOT: #define __SM4__ 1
+// CHECK_ARL_M64-NOT: #define __SM4__ 1
 // CHECK_ARLS_M64: #define __SM4__ 1
-// CHECK_SRF_M64: #define __SSE2_MATH__ 1
-// CHECK_SRF_M64: #define __SSE2__ 1
-// CHECK_SRF_M64: #define __SSE3__ 1
-// CHECK_SRF_M64: #define __SSE4_1__ 1
-// CHECK_SRF_M64: #define __SSE4_2__ 1
-// CHECK_SRF_M64: #define __SSE_MATH__ 1
-// CHECK_SRF_M64: #define __SSE__ 1
-// CHECK_SRF_M64: #define __SSSE3__ 1
-// CHECK_SRF_M64: #define __UINTR__ 1
-// CHECK_SRF_M64-NOT: #define __USERMSR__ 1
+// CHECK_ARL_M64: #define __SSE2_MATH__ 1
+// CHECK_ARL_M64: #define __SSE2__ 1
+// CHECK_ARL_M64: #define __SSE3__ 1
+// CHECK_ARL_M64: #define __SSE4_1__ 1
+// CHECK_ARL_M64: #define __SSE4_2__ 1
+// CHECK_ARL_M64: #define __SSE_MATH__ 1
+// CHECK_ARL_M64: #define __SSE__ 1
+// CHECK_ARL_M64: #define __SSSE3__ 1
+// CHECK_ARL_M64: #define __UINTR__ 1
+// CHECK_ARL_M64-NOT: #define __USERMSR__ 1
 // CHECK_ARLS_M64-NOT: #define __USERMSR__ 1
 // CHECK_PTL_M64-NOT: #define __USERMSR__ 1
 // CHECK_CWF_M64: #define __USERMSR__ 1
-// CHECK_SRF_M64: #define __VAES__ 1
-// CHECK_SRF_M64: #define __VPCLMULQDQ__ 1
-// CHECK_SRF_M64: #define __WAITPKG__ 1
-// CHECK_SRF_M64: #define __WIDEKL__ 1
-// CHECK_SRF_M64: #define __XSAVEC__ 1
-// CHECK_SRF_M64: #define __XSAVEOPT__ 1
-// CHECK_SRF_M64: #define __XSAVES__ 1
-// CHECK_SRF_M64: #define __XSAVE__ 1
-// CHECK_SRF_M64: #define __amd64 1
-// CHECK_SRF_M64: #define __amd64__ 1
-// CHECK_SRF_M64: #define __corei7 1
-// CHECK_SRF_M64: #define __corei7__ 1
-// CHECK_SRF_M64: #define __tune_corei7__ 1
-// CHECK_SRF_M64: #define __x86_64 1
-// CHECK_SRF_M64: #define __x86_64__ 1
+// CHECK_ARL_M64: #define __VAES__ 1
+// CHECK_ARL_M64: #define __VPCLMULQDQ__ 1
+// CHECK_ARL_M64: #define __WAITPKG__ 1
+// CHECK_ARL_M64: #define __WIDEKL__ 1
+// CHECK_ARL_M64: #define __XSAVEC__ 1
+// CHECK_ARL_M64: #define __XSAVEOPT__ 1
+// CHECK_ARL_M64: #define __XSAVES__ 1
+// CHECK_ARL_M64: #define __XSAVE__ 1
+// CHECK_ARL_M64: #define __amd64 1
+// CHECK_ARL_M64: #define __amd64__ 1
+// CHECK_ARL_M64: #define __corei7 1
+// CHECK_ARL_M64: #define __corei7__ 1
+// CHECK_ARL_M64: #define __tune_corei7__ 1
+// CHECK_ARL_M64: #define __x86_64 1
+// CHECK_ARL_M64: #define __x86_64__ 1
 
 // RUN: %clang -march=geode -m32 -E -dM %s -o - 2>&1 \
 // RUN:     -target i386-unknown-linux \
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index b09891652ad9..4d0bfbef794c 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -1304,16 +1304,20 @@ def ProcessorFeatures {
                                       TuningPOPCNTFalseDeps,
                                       TuningInsertVZEROUPPER];
 
-  // Sierraforest
-  list<SubtargetFeature> SRFAdditionalFeatures = [FeatureCMPCCXADD,
+  // Arrowlake
+  list<SubtargetFeature> ARLAdditionalFeatures = [FeatureCMPCCXADD,
                                                   FeatureAVXIFMA,
                                                   FeatureAVXNECONVERT,
                                                   FeatureENQCMD,
                                                   FeatureUINTR,
-                                                  FeatureCLDEMOTE,
                                                   FeatureAVXVNNIINT8];
+  list<SubtargetFeature> ARLFeatures =
+    !listconcat(ADLFeatures, ARLAdditionalFeatures);
+
+  // Sierraforest
+  list<SubtargetFeature> SRFAdditionalFeatures = [FeatureCLDEMOTE];
   list<SubtargetFeature> SRFFeatures =
-    !listconcat(ADLFeatures, SRFAdditionalFeatures);
+    !listconcat(ARLFeatures, SRFAdditionalFeatures);
 
   // Arrowlake S
   list<SubtargetFeature> ARLSAdditionalFeatures = [FeatureAVXVNNIINT16,
@@ -1321,7 +1325,7 @@ def ProcessorFeatures {
                                                    FeatureSM3,
                                                    FeatureSM4];
   list<SubtargetFeature> ARLSFeatures =
-    !listconcat(SRFFeatures, ARLSAdditionalFeatures);
+    !listconcat(ARLFeatures, ARLSAdditionalFeatures);
 
   // Pantherlake
   list<SubtargetFeature> PTLAdditionalFeatures = [FeaturePREFETCHI];
@@ -1331,9 +1335,13 @@ def ProcessorFeatures {
 
   // Clearwaterforest
   list<SubtargetFeature> CWFAdditionalFeatures = [FeaturePREFETCHI,
+                                                  FeatureAVXVNNIINT16,
+                                                  FeatureSHA512,
+                                                  FeatureSM3,
+                                                  FeatureSM4,
                                                   FeatureUSERMSR];
   list<SubtargetFeature> CWFFeatures =
-    !listconcat(ARLSFeatures, CWFAdditionalFeatures);
+    !listconcat(SRFFeatures, CWFAdditionalFeatures);
 
   // Knights Landing
   list<SubtargetFeature> KNLFeatures = [FeatureX87,
@@ -1861,7 +1869,7 @@ def : ProcModel<"raptorlake", AlderlakePModel,
 def : ProcModel<"meteorlake", AlderlakePModel,
                 ProcessorFeatures.ADLFeatures, ProcessorFeatures.ADLTuning>;
 def : ProcModel<"arrowlake", AlderlakePModel,
-                ProcessorFeatures.SRFFeatures, ProcessorFeatures.ADLTuning>;
+                ProcessorFeatures.ARLFeatures, ProcessorFeatures.ADLTuning>;
 foreach P = ["arrowlake-s", "arrowlake_s", "lunarlake"] in {
 def : ProcModel<P, AlderlakePModel,
                 ProcessorFeatures.ARLSFeatures, ProcessorFeatures.ADLTuning>;
diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp
index 4947b05cd037..94812e4e60c3 100644
--- a/llvm/lib/TargetParser/X86TargetParser.cpp
+++ b/llvm/lib/TargetParser/X86TargetParser.cpp
@@ -167,15 +167,19 @@ constexpr FeatureBitset FeaturesAlderlake =
     FeatureSERIALIZE | FeatureSHSTK | FeatureVAES | FeatureVPCLMULQDQ |
     FeatureMOVDIR64B | FeatureMOVDIRI | FeatureWAITPKG | FeatureAVXVNNI |
     FeatureHRESET | FeatureWIDEKL;
-constexpr FeatureBitset FeaturesSierraforest =
+constexpr FeatureBitset FeaturesArrowlake =
     FeaturesAlderlake | FeatureCMPCCXADD | FeatureAVXIFMA | FeatureUINTR |
-    FeatureCLDEMOTE | FeatureENQCMD | FeatureAVXNECONVERT | FeatureAVXVNNIINT8;
-constexpr FeatureBitset FeaturesArrowlakeS = FeaturesSierraforest |
-    FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 | FeatureSM4;
+    FeatureENQCMD | FeatureAVXNECONVERT | FeatureAVXVNNIINT8;
+constexpr FeatureBitset FeaturesSierraforest =
+    FeaturesArrowlake | FeatureCLDEMOTE;
+constexpr FeatureBitset FeaturesArrowlakeS =
+    FeaturesArrowlake | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
+    FeatureSM4;
 constexpr FeatureBitset FeaturesPantherlake =
     FeaturesArrowlakeS | FeaturePREFETCHI;
 constexpr FeatureBitset FeaturesClearwaterforest =
-    FeaturesArrowlakeS | FeatureUSERMSR | FeaturePREFETCHI;
+    FeaturesSierraforest | FeatureAVXVNNIINT16 | FeatureSHA512 | FeatureSM3 |
+    FeatureSM4 | FeaturePREFETCHI | FeatureUSERMSR;
 
 // Geode Processor.
 constexpr FeatureBitset FeaturesGeode =
@@ -367,7 +371,7 @@ constexpr ProcInfo Processors[] = {
   // Meteorlake microarchitecture based processors.
   { {"meteorlake"}, CK_Meteorlake, FEATURE_AVX2, FeaturesAlderlake, 'p', false },
   // Arrowlake microarchitecture based processors.
-  { {"arrowlake"}, CK_Arrowlake, FEATURE_AVX2, FeaturesSierraforest, 'p', false },
+  { {"arrowlake"}, CK_Arrowlake, FEATURE_AVX2, FeaturesArrowlake, 'p', false },
   { {"arrowlake-s"}, CK_ArrowlakeS, FEATURE_AVX2, FeaturesArrowlakeS, '\0', false },
   { {"arrowlake_s"}, CK_ArrowlakeS, FEATURE_AVX2, FeaturesArrowlakeS, 'p', true },
   // Lunarlake microarchitecture based processors.
-- 
cgit v1.2.3


From a05393a879b2950fccca66ff0e1b6c70c39838e4 Mon Sep 17 00:00:00 2001
From: Jim Lin <jim@andestech.com>
Date: Fri, 20 Jun 2025 09:06:57 +0800
Subject: [RISCV] Add symbol parsing support for XAndesPerf branch instructions
 (#137748)

This patch adds support for parsing symbols in the XAndesPerf branch
immediate instructions. The branch immediate instructions use
`R_RISCV_NDS_BRANCH_10` relocation. It uses a 10-bit PC-relative branch
offset.
---
 .../BinaryFormat/ELFRelocs/RISCV_nonstandard.def   |  4 ++
 llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp |  6 +--
 .../Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp  | 21 ++++++++
 llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h |  1 +
 .../RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp    |  2 +
 .../Target/RISCV/MCTargetDesc/RISCVFixupKinds.h    |  4 ++
 .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp      |  2 +
 llvm/lib/Target/RISCV/RISCVInstrFormats.td         | 57 +++++++++++-----------
 llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td      | 18 +++----
 llvm/test/MC/RISCV/custom_reloc.s                  | 13 +++++
 llvm/test/MC/RISCV/vendor-symbol.s                 | 19 +++++++-
 llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s | 13 +++++
 llvm/test/MC/RISCV/xandesperf-relocation.s         | 36 ++++++++++++++
 13 files changed, 153 insertions(+), 43 deletions(-)
 create mode 100644 llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
 create mode 100644 llvm/test/MC/RISCV/xandesperf-relocation.s

diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV_nonstandard.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV_nonstandard.def
index b02462ca89fd..037ca6438733 100644
--- a/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV_nonstandard.def
+++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/RISCV_nonstandard.def
@@ -26,3 +26,7 @@ ELF_RISCV_NONSTANDARD_RELOC(QUALCOMM, R_RISCV_QC_ABS20_U,    192)
 ELF_RISCV_NONSTANDARD_RELOC(QUALCOMM, R_RISCV_QC_E_BRANCH,   193)
 ELF_RISCV_NONSTANDARD_RELOC(QUALCOMM, R_RISCV_QC_E_32,       194)
 ELF_RISCV_NONSTANDARD_RELOC(QUALCOMM, R_RISCV_QC_E_CALL_PLT, 195)
+
+// Andes Nonstandard Relocations
+// Calculation: S + A - P (10-bit PC-relative branch offset)
+ELF_RISCV_NONSTANDARD_RELOC(ANDES, R_RISCV_NDS_BRANCH_10,    241)
diff --git a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
index f1d6f99ba981..45946d3efe32 100644
--- a/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
+++ b/llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp
@@ -858,10 +858,6 @@ public:
     return SignExtend64<32>(Imm);
   }
 
-  bool isSImm11Lsb0() const {
-    return isSImmPred([](int64_t Imm) { return isShiftedInt<10, 1>(Imm); });
-  }
-
   bool isSImm12() const {
     if (!isImm())
       return false;
@@ -1548,7 +1544,7 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
   case Match_InvalidSImm11:
     return generateImmOutOfRangeError(Operands, ErrorInfo, -(1 << 10),
                                       (1 << 10) - 1);
-  case Match_InvalidSImm11Lsb0:
+  case Match_InvalidBareSImm11Lsb0:
     return generateImmOutOfRangeError(
         Operands, ErrorInfo, -(1 << 10), (1 << 10) - 2,
         "immediate must be a multiple of 2 bytes in the range");
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
index 9161f23c8a95..186296944efd 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp
@@ -86,6 +86,9 @@ MCFixupKindInfo RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
       {"fixup_riscv_qc_e_32", 16, 32, 0},
       {"fixup_riscv_qc_abs20_u", 12, 20, 0},
       {"fixup_riscv_qc_e_call_plt", 0, 48, MCFixupKindInfo::FKF_IsPCRel},
+
+      // Andes fixups
+      {"fixup_riscv_nds_branch_10", 0, 32, MCFixupKindInfo::FKF_IsPCRel},
   };
   static_assert((std::size(Infos)) == RISCV::NumTargetFixupKinds,
                 "Not all fixup kinds added to Infos array");
@@ -567,6 +570,21 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value,
             (Bit15_13 << 17) | (Bit4_1 << 8) | (Bit11 << 7);
     return Value;
   }
+  case RISCV::fixup_riscv_nds_branch_10: {
+    if (!isInt<11>(Value))
+      Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
+    if (Value & 0x1)
+      Ctx.reportError(Fixup.getLoc(), "fixup value must be 2-byte aligned");
+    // Need to extract imm[10], imm[9:5], imm[4:1] from the 11-bit Value.
+    unsigned Sbit = (Value >> 10) & 0x1;
+    unsigned Hi5 = (Value >> 5) & 0x1f;
+    unsigned Lo4 = (Value >> 1) & 0xf;
+    // Inst{31} = Sbit;
+    // Inst{29-25} = Hi5;
+    // Inst{11-8} = Lo4;
+    Value = (Sbit << 31) | (Hi5 << 25) | (Lo4 << 8);
+    return Value;
+  }
   }
 }
 
@@ -702,6 +720,9 @@ void RISCVAsmBackend::maybeAddVendorReloc(const MCFragment &F,
   case RISCV::fixup_riscv_qc_e_call_plt:
     VendorIdentifier = "QUALCOMM";
     break;
+  case RISCV::fixup_riscv_nds_branch_10:
+    VendorIdentifier = "ANDES";
+    break;
   }
 
   // Create a local symbol for the vendor relocation to reference. It's fine if
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
index 6ef94fb5e93d..3d304842fac1 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h
@@ -56,6 +56,7 @@ enum {
   InstFormatQC_EB = 24,
   InstFormatQC_EJ = 25,
   InstFormatQC_ES = 26,
+  InstFormatNDS_BRANCH_10 = 27,
   InstFormatOther = 31,
 
   InstFormatMask = 31,
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
index 3c1f9450a099..8ab2c56ae317 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp
@@ -103,6 +103,8 @@ unsigned RISCVELFObjectWriter::getRelocType(const MCFixup &Fixup,
       return ELF::R_RISCV_QC_E_BRANCH;
     case RISCV::fixup_riscv_qc_e_call_plt:
       return ELF::R_RISCV_QC_E_CALL_PLT;
+    case RISCV::fixup_riscv_nds_branch_10:
+      return ELF::R_RISCV_NDS_BRANCH_10;
     }
   }
 
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
index 8d869a64cde4..b5c23772e6d8 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h
@@ -56,6 +56,10 @@ enum Fixups {
   // 32-bit fixup for symbol references in the 48-bit qc.j/qc.jal instructions
   fixup_riscv_qc_e_call_plt,
 
+  // Andes specific fixups
+  // 10-bit fixup for symbol references in the xandesperf branch instruction
+  fixup_riscv_nds_branch_10,
+
   // Used as a sentinel, must be the last
   fixup_riscv_invalid,
   NumTargetFixupKinds = fixup_riscv_invalid - FirstTargetFixupKind
diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
index 2a90552037f9..b50913be9922 100644
--- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
+++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp
@@ -647,6 +647,8 @@ uint64_t RISCVMCCodeEmitter::getImmOpValue(const MCInst &MI, unsigned OpNo,
     } else if (MIFrm == RISCVII::InstFormatQC_EJ) {
       FixupKind = RISCV::fixup_riscv_qc_e_call_plt;
       RelaxCandidate = true;
+    } else if (MIFrm == RISCVII::InstFormatNDS_BRANCH_10) {
+      FixupKind = RISCV::fixup_riscv_nds_branch_10;
     }
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVInstrFormats.td b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
index 088a6923fadb..b6b64b57b1b3 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrFormats.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrFormats.td
@@ -30,34 +30,35 @@
 class InstFormat<bits<5> val> {
   bits<5> Value = val;
 }
-def InstFormatPseudo : InstFormat<0>;
-def InstFormatR      : InstFormat<1>;
-def InstFormatR4     : InstFormat<2>;
-def InstFormatI      : InstFormat<3>;
-def InstFormatS      : InstFormat<4>;
-def InstFormatB      : InstFormat<5>;
-def InstFormatU      : InstFormat<6>;
-def InstFormatJ      : InstFormat<7>;
-def InstFormatCR     : InstFormat<8>;
-def InstFormatCI     : InstFormat<9>;
-def InstFormatCSS    : InstFormat<10>;
-def InstFormatCIW    : InstFormat<11>;
-def InstFormatCL     : InstFormat<12>;
-def InstFormatCS     : InstFormat<13>;
-def InstFormatCA     : InstFormat<14>;
-def InstFormatCB     : InstFormat<15>;
-def InstFormatCJ     : InstFormat<16>;
-def InstFormatCU     : InstFormat<17>;
-def InstFormatCLB    : InstFormat<18>;
-def InstFormatCLH    : InstFormat<19>;
-def InstFormatCSB    : InstFormat<20>;
-def InstFormatCSH    : InstFormat<21>;
-def InstFormatQC_EAI : InstFormat<22>;
-def InstFormatQC_EI  : InstFormat<23>;
-def InstFormatQC_EB  : InstFormat<24>;
-def InstFormatQC_EJ  : InstFormat<25>;
-def InstFormatQC_ES  : InstFormat<26>;
-def InstFormatOther  : InstFormat<31>;
+def InstFormatPseudo        : InstFormat<0>;
+def InstFormatR             : InstFormat<1>;
+def InstFormatR4            : InstFormat<2>;
+def InstFormatI             : InstFormat<3>;
+def InstFormatS             : InstFormat<4>;
+def InstFormatB             : InstFormat<5>;
+def InstFormatU             : InstFormat<6>;
+def InstFormatJ             : InstFormat<7>;
+def InstFormatCR            : InstFormat<8>;
+def InstFormatCI            : InstFormat<9>;
+def InstFormatCSS           : InstFormat<10>;
+def InstFormatCIW           : InstFormat<11>;
+def InstFormatCL            : InstFormat<12>;
+def InstFormatCS            : InstFormat<13>;
+def InstFormatCA            : InstFormat<14>;
+def InstFormatCB            : InstFormat<15>;
+def InstFormatCJ            : InstFormat<16>;
+def InstFormatCU            : InstFormat<17>;
+def InstFormatCLB           : InstFormat<18>;
+def InstFormatCLH           : InstFormat<19>;
+def InstFormatCSB           : InstFormat<20>;
+def InstFormatCSH           : InstFormat<21>;
+def InstFormatQC_EAI        : InstFormat<22>;
+def InstFormatQC_EI         : InstFormat<23>;
+def InstFormatQC_EB         : InstFormat<24>;
+def InstFormatQC_EJ         : InstFormat<25>;
+def InstFormatQC_ES         : InstFormat<26>;
+def InstFormatNDS_BRANCH_10 : InstFormat<27>;
+def InstFormatOther         : InstFormat<31>;
 
 
 class RISCVVConstraint<bits<3> val> {
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
index 3ba21e51e7c6..4b8d40d1429a 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXAndes.td
@@ -15,16 +15,16 @@
 //===----------------------------------------------------------------------===//
 
 // A 11-bit signed immediate where the least significant bit is zero.
-def simm11_lsb0 : Operand<OtherVT> {
-  let ParserMatchClass = SImmAsmOperand<11, "Lsb0">;
+def bare_simm11_lsb0 : Operand<OtherVT> {
+  let ParserMatchClass = BareSImmNLsb0AsmOperand<11>;
   let PrintMethod = "printBranchOperand";
   let EncoderMethod = "getImmOpValueAsrN<1>";
   let DecoderMethod = "decodeSImmOperandAndLslN<11, 1>";
   let MCOperandPredicate = [{
     int64_t Imm;
-    if (!MCOp.evaluateAsConstantImm(Imm))
-      return false;
-    return isShiftedInt<10, 1>(Imm);
+    if (MCOp.evaluateAsConstantImm(Imm))
+      return isShiftedInt<10, 1>(Imm);
+    return MCOp.isBareSymbolRef();
   }];
   let OperandType = "OPERAND_PCREL";
 }
@@ -58,8 +58,8 @@ def simm20_lsb000 : Operand<XLenVT> {
 //===----------------------------------------------------------------------===//
 
 class NDSRVInstBB<bit cs, string opcodestr>
-    : RVInst<(outs), (ins GPR:$rs1, uimmlog2xlen:$cimm, simm11_lsb0:$imm10),
-             opcodestr, "$rs1, $cimm, $imm10", [], InstFormatOther>,
+    : RVInst<(outs), (ins GPR:$rs1, uimmlog2xlen:$cimm, bare_simm11_lsb0:$imm10),
+             opcodestr, "$rs1, $cimm, $imm10", [], InstFormatNDS_BRANCH_10>,
       Sched<[WriteJmp, ReadIALU]> {
   bits<10> imm10;
   bits<5> rs1;
@@ -82,8 +82,8 @@ class NDSRVInstBB<bit cs, string opcodestr>
 }
 
 class NDSRVInstBC<bits<3> funct3, string opcodestr>
-    : RVInst<(outs), (ins GPR:$rs1, uimm7:$cimm, simm11_lsb0:$imm10),
-             opcodestr, "$rs1, $cimm, $imm10", [], InstFormatOther>,
+    : RVInst<(outs), (ins GPR:$rs1, uimm7:$cimm, bare_simm11_lsb0:$imm10),
+             opcodestr, "$rs1, $cimm, $imm10", [], InstFormatNDS_BRANCH_10>,
       Sched<[WriteJmp, ReadIALU]> {
   bits<10> imm10;
   bits<5> rs1;
diff --git a/llvm/test/MC/RISCV/custom_reloc.s b/llvm/test/MC/RISCV/custom_reloc.s
index cdb819467875..a68f71063ea9 100644
--- a/llvm/test/MC/RISCV/custom_reloc.s
+++ b/llvm/test/MC/RISCV/custom_reloc.s
@@ -48,6 +48,19 @@
   # CHECK-OBJ-NEXT: R_RISCV_VENDOR    QUALCOMM
   # CHECK-OBJ-NEXT: R_RISCV_CUSTOM192 my_bar+0x2
 
+  .reloc ., R_RISCV_VENDOR,     ANDES
+  .reloc ., R_RISCV_NDS_BRANCH_10, my_bar + 2
+  addi a1, a1, 0
+  # CHECK-ASM:      [[L3:.L[^:]+]]:
+  # CHECK-ASM-NEXT: .reloc [[L3]], R_RISCV_VENDOR, ANDES
+  # CHECK-ASM-NEXT: [[L4:.L[^:]+]]:
+  # CHECK-ASM-NEXT: .reloc [[L4]], R_RISCV_NDS_BRANCH_10, my_bar+2
+  # CHECK-ASM-NEXT: mv a1, a1
+
+  # CHECK-OBJ:      addi a1, a1, 0
+  # CHECK-OBJ-NEXT: R_RISCV_VENDOR    ANDES
+  # CHECK-OBJ-NEXT: R_RISCV_CUSTOM241 my_bar+0x2
+
   nop
   # CHECK-ASM: nop
   # CHECK-OBJ: addi zero, zero, 0x0
diff --git a/llvm/test/MC/RISCV/vendor-symbol.s b/llvm/test/MC/RISCV/vendor-symbol.s
index 7df3a3efeb64..9595f218d78f 100644
--- a/llvm/test/MC/RISCV/vendor-symbol.s
+++ b/llvm/test/MC/RISCV/vendor-symbol.s
@@ -1,4 +1,4 @@
-# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi %s \
+# RUN: llvm-mc -triple riscv32 -mattr=+experimental-xqcibi,+xandesperf %s \
 # RUN:     -filetype=obj -o - \
 # RUN:     | llvm-readelf -sr - \
 # RUN:     | FileCheck %s
@@ -18,6 +18,14 @@ QUALCOMM:
 
   qc.e.bgeui s0, 20, QUALCOMM
 
+  nds.bbc t0, 7, ANDES
+
+  .global ANDES
+ANDES:
+  nop
+
+  nds.bbs t0, 7, ANDES
+
 
 # CHECK-LABEL: Relocation section '.rela.text'
 ## Note the different values for the "Sym. Value" Field
@@ -25,11 +33,20 @@ QUALCOMM:
 # CHECK: R_RISCV_CUSTOM193 00000006 QUALCOMM + 0
 # CHECK: R_RISCV_VENDOR    00000000 QUALCOMM + 0
 # CHECK: R_RISCV_CUSTOM193 00000006 QUALCOMM + 0
+# CHECK: R_RISCV_VENDOR    00000000 ANDES + 0
+# CHECK: R_RISCV_CUSTOM241 00000014 ANDES + 0
+# CHECK: R_RISCV_VENDOR    00000000 ANDES + 0
+# CHECK: R_RISCV_CUSTOM241 00000014 ANDES + 0
 
 
 # CHECK-LABEL: Symbol table '.symtab'
 # CHECK-NOT: QUALCOMM
+# CHECK-NOT: ANDES
 # CHECK: 00000000 0 NOTYPE  LOCAL  DEFAULT ABS QUALCOMM
+# CHECK: 00000000 0 NOTYPE  LOCAL  DEFAULT ABS ANDES
 # CHECK-NOT: QUALCOMM
+# CHECK-NOT: ANDES
 # CHECK: 00000006 0 NOTYPE  GLOBAL DEFAULT   2 QUALCOMM
+# CHECK: 00000014 0 NOTYPE  GLOBAL DEFAULT   2 ANDES
 # CHECK-NOT: QUALCOMM
+# CHECK-NOT: ANDES
diff --git a/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
new file mode 100644
index 000000000000..e52f8129129d
--- /dev/null
+++ b/llvm/test/MC/RISCV/xandesperf-fixups-diagnostics.s
@@ -0,0 +1,13 @@
+# RUN: not llvm-mc -triple riscv32 -filetype obj -mattr=+xandesperf < %s -o /dev/null 2>&1 | FileCheck %s
+
+  nds.bbc t0, 7, far_distant # CHECK: :[[@LINE]]:3: error: fixup value out of range
+  nds.bbc t0, 7, unaligned # CHECK: :[[@LINE]]:3: error: fixup value must be 2-byte aligned
+
+  .byte 0
+unaligned:
+  .byte 0
+  .byte 0
+  .byte 0
+
+  .space 1<<10
+far_distant:
diff --git a/llvm/test/MC/RISCV/xandesperf-relocation.s b/llvm/test/MC/RISCV/xandesperf-relocation.s
new file mode 100644
index 000000000000..4df75f04dbfe
--- /dev/null
+++ b/llvm/test/MC/RISCV/xandesperf-relocation.s
@@ -0,0 +1,36 @@
+# RUN: llvm-mc -triple riscv32 -mattr=+xandesperf -M no-aliases < %s -show-encoding \
+# RUN:     | FileCheck -check-prefix=ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv32 -mattr=+xandesperf < %s \
+# RUN:     | llvm-objdump -dr --mattr=+xandesperf - \
+# RUN:     | FileCheck -check-prefix=OBJ %s
+# RUN: llvm-mc -triple riscv64 -mattr=+xandesperf -M no-aliases < %s -show-encoding \
+# RUN:     | FileCheck -check-prefix=ASM %s
+# RUN: llvm-mc -filetype=obj -triple riscv64 -mattr=+xandesperf < %s \
+# RUN:     | llvm-objdump -dr --mattr=+xandesperf - \
+# RUN:     | FileCheck -check-prefix=OBJ %s
+
+.long foo
+
+# ASM: nds.bbc t0, 7, foo
+# OBJ: nds.bbc t0, 0x7, 0x4 <.text+0x4>
+# OBJ-NEXT: R_RISCV_VENDOR ANDES{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM241 foo{{$}}
+nds.bbc t0, 7, foo
+
+# ASM: nds.bbs t0, 7, foo
+# OBJ-NEXT: nds.bbs t0, 0x7, 0x8 <.text+0x8>
+# OBJ-NEXT: R_RISCV_VENDOR ANDES{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM241 foo{{$}}
+nds.bbs t0, 7, foo
+
+# ASM: nds.beqc t0, 7, foo
+# OBJ-NEXT: nds.beqc t0, 0x7, 0xc <.text+0xc>
+# OBJ-NEXT: R_RISCV_VENDOR ANDES{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM241 foo{{$}}
+nds.beqc t0, 7, foo
+
+# ASM: nds.bnec t0, 7, foo
+# OBJ-NEXT: nds.bnec t0, 0x7, 0x10 <.text+0x10>
+# OBJ-NEXT: R_RISCV_VENDOR ANDES{{$}}
+# OBJ-NEXT: R_RISCV_CUSTOM241 foo{{$}}
+nds.bnec t0, 7, foo
-- 
cgit v1.2.3


From 513bcf6d012f7f9483af784de8487ee04cb9971a Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Thu, 19 Jun 2025 18:38:58 -0700
Subject: [yaml2obj] Report error when the input filename does not exist

I invoked yaml2obj with a mistyped filename and received no error
message. I nearly thought the program had succeeded, but the shell's
exit code prompt tipped me off.

Pull Request: https://github.com/llvm/llvm-project/pull/144835
---
 llvm/test/tools/yaml2obj/basic.test | 4 ++++
 llvm/tools/yaml2obj/yaml2obj.cpp    | 9 ++++++---
 2 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100644 llvm/test/tools/yaml2obj/basic.test

diff --git a/llvm/test/tools/yaml2obj/basic.test b/llvm/test/tools/yaml2obj/basic.test
new file mode 100644
index 000000000000..d673accfee88
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/basic.test
@@ -0,0 +1,4 @@
+# Test case when the input file does not exist.
+RUN: not yaml2obj %t.blah 2>&1 | FileCheck -DMSG=%errc_ENOENT --check-prefix=ENOENT %s
+
+ENOENT: yaml2obj: error: {{.*}}.blah: [[MSG]]
diff --git a/llvm/tools/yaml2obj/yaml2obj.cpp b/llvm/tools/yaml2obj/yaml2obj.cpp
index 45aa3828311f..dcd6dfcd3de2 100644
--- a/llvm/tools/yaml2obj/yaml2obj.cpp
+++ b/llvm/tools/yaml2obj/yaml2obj.cpp
@@ -117,8 +117,9 @@ int main(int argc, char **argv) {
       argc, argv, "Create an object file from a YAML description", nullptr,
       nullptr, /*LongOptionsUseDoubleDash=*/true);
 
-  auto ErrHandler = [](const Twine &Msg) {
-    WithColor::error(errs(), "yaml2obj") << Msg << "\n";
+  constexpr StringRef ProgName = "yaml2obj";
+  auto ErrHandler = [&](const Twine &Msg) {
+    WithColor::error(errs(), ProgName) << Msg << "\n";
   };
 
   std::error_code EC;
@@ -131,8 +132,10 @@ int main(int argc, char **argv) {
 
   ErrorOr<std::unique_ptr<MemoryBuffer>> Buf =
       MemoryBuffer::getFileOrSTDIN(Input, /*IsText=*/true);
-  if (!Buf)
+  if (std::error_code EC = Buf.getError()) {
+    WithColor::error(errs(), ProgName) << Input << ": " << EC.message() << '\n';
     return 1;
+  }
 
   std::optional<std::string> Buffer =
       preprocess(Buf.get()->getBuffer(), ErrHandler);
-- 
cgit v1.2.3


From 4f991cc99523e4bb7a0d96cee9f5c3a64bf2bc8e Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Thu, 19 Jun 2025 20:48:07 -0500
Subject: [lldb-dap] Make connection URLs match lldb (#144770)

Use the same scheme as ConnectionFileDescriptor::Connect and use
"listen" and "accept". Addresses feedback from a Pavel in a different PR
[1].

[1] https://github.com/llvm/llvm-project/pull/143628#discussion_r2152225200
---
 lldb/include/lldb/Host/Socket.h                    |  9 ++++++
 lldb/source/Host/common/Socket.cpp                 | 32 ++++++++++++++++++++--
 .../API/tools/lldb-dap/server/TestDAP_server.py    |  6 ++--
 lldb/tools/lldb-dap/Options.td                     |  4 +--
 lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts      |  2 +-
 lldb/tools/lldb-dap/tool/lldb-dap.cpp              | 29 +++++++++++++-------
 6 files changed, 63 insertions(+), 19 deletions(-)

diff --git a/lldb/include/lldb/Host/Socket.h b/lldb/include/lldb/Host/Socket.h
index 4585eac12efb..c313aa4f6d26 100644
--- a/lldb/include/lldb/Host/Socket.h
+++ b/lldb/include/lldb/Host/Socket.h
@@ -74,6 +74,11 @@ public:
     ProtocolUnixAbstract
   };
 
+  enum SocketMode {
+    ModeAccept,
+    ModeConnect,
+  };
+
   struct HostAndPort {
     std::string hostname;
     uint16_t port;
@@ -83,6 +88,10 @@ public:
     }
   };
 
+  using ProtocolModePair = std::pair<SocketProtocol, SocketMode>;
+  static std::optional<ProtocolModePair>
+  GetProtocolAndMode(llvm::StringRef scheme);
+
   static const NativeSocket kInvalidSocketValue;
 
   ~Socket() override;
diff --git a/lldb/source/Host/common/Socket.cpp b/lldb/source/Host/common/Socket.cpp
index 76f74401ac4d..5c5cd653c3d9 100644
--- a/lldb/source/Host/common/Socket.cpp
+++ b/lldb/source/Host/common/Socket.cpp
@@ -271,7 +271,8 @@ Socket::UdpConnect(llvm::StringRef host_and_port) {
   return UDPSocket::CreateConnected(host_and_port);
 }
 
-llvm::Expected<Socket::HostAndPort> Socket::DecodeHostAndPort(llvm::StringRef host_and_port) {
+llvm::Expected<Socket::HostAndPort>
+Socket::DecodeHostAndPort(llvm::StringRef host_and_port) {
   static llvm::Regex g_regex("([^:]+|\\[[0-9a-fA-F:]+.*\\]):([0-9]+)");
   HostAndPort ret;
   llvm::SmallVector<llvm::StringRef, 3> matches;
@@ -347,8 +348,8 @@ Status Socket::Write(const void *buf, size_t &num_bytes) {
               ", src = %p, src_len = %" PRIu64 ", flags = 0) => %" PRIi64
               " (error = %s)",
               static_cast<void *>(this), static_cast<uint64_t>(m_socket), buf,
-              static_cast<uint64_t>(src_len),
-              static_cast<int64_t>(bytes_sent), error.AsCString());
+              static_cast<uint64_t>(src_len), static_cast<int64_t>(bytes_sent),
+              error.AsCString());
   }
 
   return error;
@@ -476,3 +477,28 @@ llvm::raw_ostream &lldb_private::operator<<(llvm::raw_ostream &OS,
                                             const Socket::HostAndPort &HP) {
   return OS << '[' << HP.hostname << ']' << ':' << HP.port;
 }
+
+std::optional<Socket::ProtocolModePair>
+Socket::GetProtocolAndMode(llvm::StringRef scheme) {
+  // Keep in sync with ConnectionFileDescriptor::Connect.
+  return llvm::StringSwitch<std::optional<ProtocolModePair>>(scheme)
+      .Case("listen", ProtocolModePair{SocketProtocol::ProtocolTcp,
+                                       SocketMode::ModeAccept})
+      .Cases("accept", "unix-accept",
+             ProtocolModePair{SocketProtocol::ProtocolUnixDomain,
+                              SocketMode::ModeAccept})
+      .Case("unix-abstract-accept",
+            ProtocolModePair{SocketProtocol::ProtocolUnixAbstract,
+                             SocketMode::ModeAccept})
+      .Cases("connect", "tcp-connect",
+             ProtocolModePair{SocketProtocol::ProtocolTcp,
+                              SocketMode::ModeConnect})
+      .Case("udp", ProtocolModePair{SocketProtocol::ProtocolTcp,
+                                    SocketMode::ModeConnect})
+      .Case("unix-connect", ProtocolModePair{SocketProtocol::ProtocolUnixDomain,
+                                             SocketMode::ModeConnect})
+      .Case("unix-abstract-connect",
+            ProtocolModePair{SocketProtocol::ProtocolUnixAbstract,
+                             SocketMode::ModeConnect})
+      .Default(std::nullopt);
+}
diff --git a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
index ed17044a220d..592a4cfb0a88 100644
--- a/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
+++ b/lldb/test/API/tools/lldb-dap/server/TestDAP_server.py
@@ -54,7 +54,7 @@ class TestDAP_server(lldbdap_testcase.DAPTestCaseBase):
         Test launching a binary with a lldb-dap in server mode on a specific port.
         """
         self.build()
-        (_, connection) = self.start_server(connection="tcp://localhost:0")
+        (_, connection) = self.start_server(connection="listen://localhost:0")
         self.run_debug_session(connection, "Alice")
         self.run_debug_session(connection, "Bob")
 
@@ -72,7 +72,7 @@ class TestDAP_server(lldbdap_testcase.DAPTestCaseBase):
         self.addTearDownHook(cleanup)
 
         self.build()
-        (_, connection) = self.start_server(connection="unix://" + name)
+        (_, connection) = self.start_server(connection="accept://" + name)
         self.run_debug_session(connection, "Alice")
         self.run_debug_session(connection, "Bob")
 
@@ -82,7 +82,7 @@ class TestDAP_server(lldbdap_testcase.DAPTestCaseBase):
         Test launching a binary with lldb-dap in server mode and shutting down the server while the debug session is still active.
         """
         self.build()
-        (process, connection) = self.start_server(connection="tcp://localhost:0")
+        (process, connection) = self.start_server(connection="listen://localhost:0")
         self.dap_server = dap_server.DebugAdapterServer(
             connection=connection,
         )
diff --git a/lldb/tools/lldb-dap/Options.td b/lldb/tools/lldb-dap/Options.td
index aecf91797ac7..867753e9294a 100644
--- a/lldb/tools/lldb-dap/Options.td
+++ b/lldb/tools/lldb-dap/Options.td
@@ -28,8 +28,8 @@ def connection
       MetaVarName<"<connection>">,
       HelpText<
           "Communicate with the lldb-dap tool over the specified connection. "
-          "Connections are specified like 'tcp://[host]:port' or "
-          "'unix:///path'.">;
+          "Connections are specified like 'listen://[host]:port' or "
+          "'accept:///path'.">;
 
 def launch_target: S<"launch-target">,
   MetaVarName<"<target>">,
diff --git a/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts b/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
index f40dbf049a4b..79573ec7342b 100644
--- a/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
+++ b/lldb/tools/lldb-dap/src-ts/lldb-dap-server.ts
@@ -26,7 +26,7 @@ export class LLDBDapServer implements vscode.Disposable {
     args: string[],
     options?: child_process.SpawnOptionsWithoutStdio,
   ): Promise<{ host: string; port: number } | undefined> {
-    const dapArgs = [...args, "--connection", "connect://localhost:0"];
+    const dapArgs = [...args, "--connection", "listen://localhost:0" ];
     if (!(await this.shouldContinueStartup(dapPath, dapArgs))) {
       return undefined;
     }
diff --git a/lldb/tools/lldb-dap/tool/lldb-dap.cpp b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
index 9b9de5e21a74..2799d10ae84b 100644
--- a/lldb/tools/lldb-dap/tool/lldb-dap.cpp
+++ b/lldb/tools/lldb-dap/tool/lldb-dap.cpp
@@ -127,7 +127,7 @@ EXAMPLES:
   parent over stdio. Passing a --connection URI will cause lldb-dap to listen
   for a connection in the specified mode.
 
-    lldb-dap --connection connection://localhost:<port>
+    lldb-dap --connection listen://localhost:<port>
 
   Passing --wait-for-debugger will pause the process at startup and wait for a
   debugger to attach to the process.
@@ -226,23 +226,32 @@ static llvm::Expected<std::pair<Socket::SocketProtocol, std::string>>
 validateConnection(llvm::StringRef conn) {
   auto uri = lldb_private::URI::Parse(conn);
 
-  if (uri && (uri->scheme == "tcp" || uri->scheme == "connect" ||
-              !uri->hostname.empty() || uri->port)) {
+  auto make_error = [conn]() -> llvm::Error {
+    return llvm::createStringError(
+        "Unsupported connection specifier, expected 'accept:///path' or "
+        "'listen://[host]:port', got '%s'.",
+        conn.str().c_str());
+  };
+
+  if (!uri)
+    return make_error();
+
+  std::optional<Socket::ProtocolModePair> protocol_and_mode =
+      Socket::GetProtocolAndMode(uri->scheme);
+  if (!protocol_and_mode || protocol_and_mode->second != Socket::ModeAccept)
+    return make_error();
+
+  if (protocol_and_mode->first == Socket::ProtocolTcp) {
     return std::make_pair(
         Socket::ProtocolTcp,
         formatv("[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
                 uri->port.value_or(0)));
   }
 
-  if (uri && (uri->scheme == "unix" || uri->scheme == "unix-connect" ||
-              uri->path != "/")) {
+  if (protocol_and_mode->first == Socket::ProtocolUnixDomain)
     return std::make_pair(Socket::ProtocolUnixDomain, uri->path.str());
-  }
 
-  return llvm::createStringError(
-      "Unsupported connection specifier, expected 'unix-connect:///path' or "
-      "'connect://[host]:port', got '%s'.",
-      conn.str().c_str());
+  return make_error();
 }
 
 static llvm::Error
-- 
cgit v1.2.3


From 50c5ecd35402dc734f2a462df5532e77a5ce12b2 Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Fri, 20 Jun 2025 10:33:32 +0800
Subject: [NFC] [Serialization] Remove a useless comment

The comments belongs to other WIP patches.
---
 clang/include/clang/Serialization/ASTReader.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h
index 866986dcbf76..be1c6e081759 100644
--- a/clang/include/clang/Serialization/ASTReader.h
+++ b/clang/include/clang/Serialization/ASTReader.h
@@ -379,8 +379,7 @@ struct VisibleLookupBlockOffsets {
   uint64_t TULocalOffset = 0;
 
   operator bool() const {
-    return VisibleOffset || ModuleLocalOffset || // ModuleUnitLocalOffset ||
-           TULocalOffset;
+    return VisibleOffset || ModuleLocalOffset || TULocalOffset;
   }
 };
 
-- 
cgit v1.2.3


From bb51c5d4b85a655e8c90247a6678e507402703a4 Mon Sep 17 00:00:00 2001
From: Ming Yan <99472920+NexMing@users.noreply.github.com>
Date: Fri, 20 Jun 2025 10:58:02 +0800
Subject: [SDPatternMatch] Add m_Poison matcher (#144860)

Add SDPatternMatch matcher and unit test coverage for ISD::POISON opcode

e.g.
```
m_InsertElt(m_Poison(), m_Value(), m_Zero())
```
---
 llvm/include/llvm/CodeGen/SDPatternMatch.h              | 2 ++
 llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp | 5 +++++
 2 files changed, 7 insertions(+)

diff --git a/llvm/include/llvm/CodeGen/SDPatternMatch.h b/llvm/include/llvm/CodeGen/SDPatternMatch.h
index d413227c4d96..9eb6dd45f912 100644
--- a/llvm/include/llvm/CodeGen/SDPatternMatch.h
+++ b/llvm/include/llvm/CodeGen/SDPatternMatch.h
@@ -142,6 +142,8 @@ inline Opcode_match m_Opc(unsigned Opcode) { return Opcode_match(Opcode); }
 
 inline Opcode_match m_Undef() { return Opcode_match(ISD::UNDEF); }
 
+inline Opcode_match m_Poison() { return Opcode_match(ISD::POISON); }
+
 template <unsigned NumUses, typename Pattern> struct NUses_match {
   Pattern P;
 
diff --git a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
index 2162588aadfd..2b1fa75a1475 100644
--- a/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
+++ b/llvm/unittests/CodeGen/SelectionDAGPatternMatchTest.cpp
@@ -493,6 +493,11 @@ TEST_F(SelectionDAGPatternMatchTest, matchConstants) {
   SDValue UndefVInt32VT = DAG->getUNDEF(VInt32VT);
   EXPECT_TRUE(sd_match(UndefInt32VT, m_Undef()));
   EXPECT_TRUE(sd_match(UndefVInt32VT, m_Undef()));
+
+  SDValue PoisonInt32VT = DAG->getPOISON(Int32VT);
+  SDValue PoisonVInt32VT = DAG->getPOISON(VInt32VT);
+  EXPECT_TRUE(sd_match(PoisonInt32VT, m_Poison()));
+  EXPECT_TRUE(sd_match(PoisonVInt32VT, m_Poison()));
 }
 
 TEST_F(SelectionDAGPatternMatchTest, patternCombinators) {
-- 
cgit v1.2.3


From 64fe3236476a9a85977abf5489414bbb2de2109c Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 21:31:26 -0700
Subject: [llvm] Migrate away from ArrayRef(std::nullopt) (NFC) (#144967)

ArrayRef has a constructor that accepts std::nullopt.  This
constructor dates back to the days when we still had llvm::Optional.

Since the use of std::nullopt outside the context of std::optional is
kind of abuse and not intuitive to new comers, I would like to move
away from the constructor and eventually remove it.

This patch takes care of the llvm side of the migration.
---
 llvm/include/llvm/CodeGen/BasicTTIImpl.h        |  4 ++--
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 10 +++++-----
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp  |  3 +--
 3 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index 3b87978fe3fa..90a75c3d352e 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -2376,8 +2376,8 @@ public:
                                           CostKind, 1, nullptr, nullptr);
       Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, SearchTy,
                                           CostKind, 0, nullptr, nullptr);
-      Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, SearchTy, std::nullopt,
-                                      CostKind, 0, nullptr);
+      Cost += thisT()->getShuffleCost(TTI::SK_Broadcast, SearchTy, {}, CostKind,
+                                      0, nullptr);
       Cost += thisT()->getCmpSelInstrCost(BinaryOperator::ICmp, SearchTy, RetTy,
                                           CmpInst::ICMP_EQ, CostKind);
       Cost +=
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 4551a365a696..5eef2497cf90 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -5994,7 +5994,7 @@ static bool isMaskedLoadCompress(
       InstructionCost InterleavedCost =
           VectorGEPCost + TTI.getInterleavedMemoryOpCost(
                               Instruction::Load, AlignedLoadVecTy,
-                              CompressMask[1], std::nullopt, CommonAlignment,
+                              CompressMask[1], {}, CommonAlignment,
                               LI->getPointerAddressSpace(), CostKind, IsMasked);
       if (InterleavedCost < GatherCost) {
         InterleaveFactor = CompressMask[1];
@@ -13561,7 +13561,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
       case TreeEntry::Vectorize:
         if (unsigned Factor = E->getInterleaveFactor()) {
           VecLdCost = TTI->getInterleavedMemoryOpCost(
-              Instruction::Load, VecTy, Factor, std::nullopt, LI0->getAlign(),
+              Instruction::Load, VecTy, Factor, {}, LI0->getAlign(),
               LI0->getPointerAddressSpace(), CostKind);
 
         } else {
@@ -13602,7 +13602,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
         Align CommonAlignment = LI0->getAlign();
         if (InterleaveFactor) {
           VecLdCost = TTI->getInterleavedMemoryOpCost(
-              Instruction::Load, LoadVecTy, InterleaveFactor, std::nullopt,
+              Instruction::Load, LoadVecTy, InterleaveFactor, {},
               CommonAlignment, LI0->getPointerAddressSpace(), CostKind);
         } else if (IsMasked) {
           VecLdCost = TTI->getMaskedMemoryOpCost(
@@ -13677,8 +13677,8 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef<Value *> VectorizedVals,
                  "No reused shuffles expected");
           CommonCost = 0;
           VecStCost = TTI->getInterleavedMemoryOpCost(
-              Instruction::Store, VecTy, Factor, std::nullopt,
-              BaseSI->getAlign(), BaseSI->getPointerAddressSpace(), CostKind);
+              Instruction::Store, VecTy, Factor, {}, BaseSI->getAlign(),
+              BaseSI->getPointerAddressSpace(), CostKind);
         } else {
           TTI::OperandValueInfo OpInfo = getOperandInfo(E->getOperand(0));
           VecStCost = TTI->getMemoryOpCost(
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index 22861eb1c7df..f45ce46763c5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3478,8 +3478,7 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
 
   return Cost + IG->getNumMembers() *
                     Ctx.TTI.getShuffleCost(TargetTransformInfo::SK_Reverse,
-                                           VectorTy, std::nullopt, Ctx.CostKind,
-                                           0);
+                                           VectorTy, {}, Ctx.CostKind, 0);
 }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-- 
cgit v1.2.3


From 69974658f079cec82a9fc13dd4993ab1e072c811 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Thu, 19 Jun 2025 22:52:51 -0700
Subject: [AMDGPU] Initial support for gfx1250 target. (#144965)

This is just a stub for now.
---
 clang/include/clang/Basic/OffloadArch.h            |  1 +
 clang/lib/Basic/OffloadArch.cpp                    |  1 +
 clang/lib/Basic/Targets/NVPTX.cpp                  |  1 +
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp           |  1 +
 clang/test/CodeGenOpenCL/amdgpu-features.cl        |  2 +
 clang/test/Driver/amdgpu-macros.cl                 | 11 ++++-
 clang/test/Driver/amdgpu-mcpu.cl                   |  2 +
 clang/test/Misc/target-invalid-cpu-note/amdgcn.c   |  1 +
 clang/test/Misc/target-invalid-cpu-note/nvptx.c    |  1 +
 llvm/docs/AMDGPUUsage.rst                          |  9 +++-
 llvm/include/llvm/BinaryFormat/ELF.h               |  2 +-
 llvm/include/llvm/TargetParser/TargetParser.h      |  3 +-
 llvm/lib/Object/ELFObjectFile.cpp                  |  2 +
 llvm/lib/ObjectYAML/ELFYAML.cpp                    |  1 +
 llvm/lib/Target/AMDGPU/AMDGPU.td                   | 53 ++++++++++++++++++++++
 llvm/lib/Target/AMDGPU/GCNProcessors.td            |  4 ++
 llvm/lib/Target/AMDGPU/GCNSubtarget.h              |  1 +
 .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp   |  2 +
 llvm/lib/TargetParser/TargetParser.cpp             | 30 ++++++++++++
 .../test/CodeGen/AMDGPU/directive-amdgcn-target.ll |  2 +
 llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll  |  2 +
 llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml |  7 +++
 .../tools/llvm-objdump/ELF/AMDGPU/subtarget.ll     |  5 ++
 .../tools/llvm-readobj/ELF/AMDGPU/elf-headers.test |  9 ++++
 llvm/tools/llvm-readobj/ELFDumper.cpp              |  1 +
 25 files changed, 149 insertions(+), 5 deletions(-)

diff --git a/clang/include/clang/Basic/OffloadArch.h b/clang/include/clang/Basic/OffloadArch.h
index 99b1024b9d0d..4dda3ec2216f 100644
--- a/clang/include/clang/Basic/OffloadArch.h
+++ b/clang/include/clang/Basic/OffloadArch.h
@@ -98,6 +98,7 @@ enum class OffloadArch {
   GFX12_GENERIC,
   GFX1200,
   GFX1201,
+  GFX1250,
   AMDGCNSPIRV,
   Generic, // A processor model named 'generic' if the target backend defines a
            // public one.
diff --git a/clang/lib/Basic/OffloadArch.cpp b/clang/lib/Basic/OffloadArch.cpp
index a019f0ac18c8..dce9ffaedb90 100644
--- a/clang/lib/Basic/OffloadArch.cpp
+++ b/clang/lib/Basic/OffloadArch.cpp
@@ -86,6 +86,7 @@ static const OffloadArchToStringMap ArchNames[] = {
     {OffloadArch::GFX12_GENERIC, "gfx12-generic", "compute_amdgcn"},
     GFX(1200), // gfx1200
     GFX(1201), // gfx1201
+    GFX(1250), // gfx1250
     {OffloadArch::AMDGCNSPIRV, "amdgcnspirv", "compute_amdgcn"},
     // Intel CPUs
     {OffloadArch::GRANITERAPIDS, "graniterapids", ""},
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 3235bf2e710d..54b39fd072a8 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -238,6 +238,7 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts,
       case OffloadArch::GFX12_GENERIC:
       case OffloadArch::GFX1200:
       case OffloadArch::GFX1201:
+      case OffloadArch::GFX1250:
       case OffloadArch::AMDGCNSPIRV:
       case OffloadArch::Generic:
       case OffloadArch::GRANITERAPIDS:
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 9e27e634676d..2bc9cd549f01 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -2331,6 +2331,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective(const OMPRequiresDecl *D) {
       case OffloadArch::GFX12_GENERIC:
       case OffloadArch::GFX1200:
       case OffloadArch::GFX1201:
+      case OffloadArch::GFX1250:
       case OffloadArch::AMDGCNSPIRV:
       case OffloadArch::Generic:
       case OffloadArch::GRANITERAPIDS:
diff --git a/clang/test/CodeGenOpenCL/amdgpu-features.cl b/clang/test/CodeGenOpenCL/amdgpu-features.cl
index b94e1e76c7a2..730ed47f0b0c 100644
--- a/clang/test/CodeGenOpenCL/amdgpu-features.cl
+++ b/clang/test/CodeGenOpenCL/amdgpu-features.cl
@@ -52,6 +52,7 @@
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1153 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1153 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1200 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1200 %s
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1201 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1201 %s
+// RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1250 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1250 %s
 
 // RUN: %clang_cc1 -triple amdgcn -target-cpu gfx1103 -target-feature +wavefrontsize64 -emit-llvm -o - %s | FileCheck --check-prefix=GFX1103-W64 %s
 
@@ -107,6 +108,7 @@
 // GFX1153: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1200: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
 // GFX1201: "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+ci-insts,+dl-insts,+dot10-insts,+dot11-insts,+dot12-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize32"
+// GFX1250: "target-features"="+16-bit-insts,+ashr-pk-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-buffer-pk-add-bf16-inst,+atomic-ds-pk-add-16-insts,+atomic-fadd-rtn-insts,+atomic-flat-pk-add-16-insts,+atomic-global-pk-add-bf16-inst,+bitop3-insts,+ci-insts,+dl-insts,+dot7-insts,+dot8-insts,+dpp,+fp8-conversion-insts,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx12-insts,+gfx1250-insts,+gfx8-insts,+gfx9-insts,+permlane16-swap,+prng-inst,+setprio-inc-wg-inst,+wavefrontsize32"
 
 // GFX1103-W64: "target-features"="+16-bit-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot10-insts,+dot12-insts,+dot5-insts,+dot7-insts,+dot8-insts,+dot9-insts,+dpp,+gfx10-3-insts,+gfx10-insts,+gfx11-insts,+gfx8-insts,+gfx9-insts,+wavefrontsize64"
 
diff --git a/clang/test/Driver/amdgpu-macros.cl b/clang/test/Driver/amdgpu-macros.cl
index 35dc190761ca..a60593f2ab9e 100644
--- a/clang/test/Driver/amdgpu-macros.cl
+++ b/clang/test/Driver/amdgpu-macros.cl
@@ -130,6 +130,7 @@
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1153 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1153 -DFAMILY=GFX11
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1200 -DFAMILY=GFX12
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1201 -DFAMILY=GFX12
+// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1250 %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=32 -DCPU=gfx1250 -DFAMILY=GFX12
 
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx9-generic %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=64 -DCPU=gfx9_generic -DFAMILY=GFX9
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx9-4-generic %s 2>&1 | FileCheck --check-prefixes=ARCH-GCN,FAST_FMAF %s -DWAVEFRONT_SIZE=64 -DCPU=gfx9_4_generic -DFAMILY=GFX9
@@ -177,13 +178,19 @@
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx906 -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx906 -mno-cumode \
-// RUN:   %s 2>&1 | FileCheck --check-prefixes=CUMODE-ON,WARN-CUMODE %s
+// RUN:   %s 2>&1 | FileCheck -DMCPU=gfx906 --check-prefixes=CUMODE-ON,WARN-CUMODE %s
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1030 \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1030 -mcumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
 // RUN: %clang -E -dM -target amdgcn -mcpu=gfx1030 -mno-cumode \
 // RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-OFF %s
-// WARN-CUMODE-DAG: warning: ignoring '-mno-cumode' option as it is not currently supported for processor 'gfx906' [-Woption-ignored]
+// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1250 \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
+// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1250 -mcumode \
+// RUN:   %s 2>&1 | FileCheck --check-prefix=CUMODE-ON %s
+// RUN: %clang -E -dM -target amdgcn -mcpu=gfx1250 -mno-cumode \
+// RUN:   %s 2>&1 | FileCheck -DMCPU=gfx1250 --check-prefixes=CUMODE-ON,WARN-CUMODE %s
+// WARN-CUMODE-DAG: warning: ignoring '-mno-cumode' option as it is not currently supported for processor '[[MCPU]]' [-Woption-ignored]
 // CUMODE-ON-DAG: #define __AMDGCN_CUMODE__ 1
 // CUMODE-OFF-DAG: #define __AMDGCN_CUMODE__ 0
diff --git a/clang/test/Driver/amdgpu-mcpu.cl b/clang/test/Driver/amdgpu-mcpu.cl
index ad5fd8ebaa6a..6d302e4c59ad 100644
--- a/clang/test/Driver/amdgpu-mcpu.cl
+++ b/clang/test/Driver/amdgpu-mcpu.cl
@@ -115,6 +115,7 @@
 // RUN: %clang -### -target amdgcn -mcpu=gfx1153 %s 2>&1 | FileCheck --check-prefix=GFX1153 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefix=GFX1200 %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx1201 %s 2>&1 | FileCheck --check-prefix=GFX1201 %s
+// RUN: %clang -### -target amdgcn -mcpu=gfx1250 %s 2>&1 | FileCheck --check-prefix=GFX1250 %s
 
 // RUN: %clang -### -target amdgcn -mcpu=gfx9-generic %s 2>&1 | FileCheck --check-prefix=GFX9_GENERIC %s
 // RUN: %clang -### -target amdgcn -mcpu=gfx9-4-generic %s 2>&1 | FileCheck --check-prefix=GFX9_4_GENERIC %s
@@ -169,6 +170,7 @@
 // GFX1153:   "-target-cpu" "gfx1153"
 // GFX1200:   "-target-cpu" "gfx1200"
 // GFX1201:   "-target-cpu" "gfx1201"
+// GFX1250:   "-target-cpu" "gfx1250"
 
 // GFX9_GENERIC:      "-target-cpu" "gfx9-generic"
 // GFX9_4_GENERIC:    "-target-cpu" "gfx9-4-generic"
diff --git a/clang/test/Misc/target-invalid-cpu-note/amdgcn.c b/clang/test/Misc/target-invalid-cpu-note/amdgcn.c
index 9ef44b2bb403..352658b6fb38 100644
--- a/clang/test/Misc/target-invalid-cpu-note/amdgcn.c
+++ b/clang/test/Misc/target-invalid-cpu-note/amdgcn.c
@@ -68,6 +68,7 @@
 // CHECK-SAME: {{^}}, gfx1153
 // CHECK-SAME: {{^}}, gfx1200
 // CHECK-SAME: {{^}}, gfx1201
+// CHECK-SAME: {{^}}, gfx1250
 // CHECK-SAME: {{^}}, gfx9-generic
 // CHECK-SAME: {{^}}, gfx10-1-generic
 // CHECK-SAME: {{^}}, gfx10-3-generic
diff --git a/clang/test/Misc/target-invalid-cpu-note/nvptx.c b/clang/test/Misc/target-invalid-cpu-note/nvptx.c
index 6675a1ecc34b..b5209ffc5f0a 100644
--- a/clang/test/Misc/target-invalid-cpu-note/nvptx.c
+++ b/clang/test/Misc/target-invalid-cpu-note/nvptx.c
@@ -83,5 +83,6 @@
 // CHECK-SAME: {{^}}, gfx12-generic
 // CHECK-SAME: {{^}}, gfx1200
 // CHECK-SAME: {{^}}, gfx1201
+// CHECK-SAME: {{^}}, gfx1250
 // CHECK-SAME: {{^}}, amdgcnspirv
 // CHECK-SAME: {{$}}
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index ed3e4c8513e2..2fb4f5389fc7 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -531,6 +531,13 @@ Every processor supports every OS ABI (see :ref:`amdgpu-os`) with the following
                                                                         work-item                       Add product
                                                                         IDs                             names.
 
+     ``gfx1250``                 ``amdgcn``   APU                     - Architected                   *TBA*
+                                                                        flat
+                                                                        scratch                       .. TODO::
+                                                                      - Packed
+                                                                        work-item                       Add product
+                                                                        IDs                             names.
+
      =========== =============== ============ ===== ================= =============== =============== ======================
 
 Generic processors allow execution of a single code object on any of the processors that
@@ -2265,7 +2272,7 @@ The AMDGPU backend uses the following ELF header:
      ``EF_AMDGPU_MACH_AMDGCN_GFX1101``          0x046      ``gfx1101``
      ``EF_AMDGPU_MACH_AMDGCN_GFX1102``          0x047      ``gfx1102``
      ``EF_AMDGPU_MACH_AMDGCN_GFX1200``          0x048      ``gfx1200``
-     *reserved*                                 0x049      Reserved.
+     ``EF_AMDGPU_MACH_AMDGCN_GFX1250``          0x049      ``gfx1250``
      ``EF_AMDGPU_MACH_AMDGCN_GFX1151``          0x04a      ``gfx1151``
      *reserved*                                 0x04b      Reserved.
      ``EF_AMDGPU_MACH_AMDGCN_GFX942``           0x04c      ``gfx942``
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index dcf1f1c6d7b2..f5f236cf9806 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -835,7 +835,7 @@ enum : unsigned {
   EF_AMDGPU_MACH_AMDGCN_GFX1101         = 0x046,
   EF_AMDGPU_MACH_AMDGCN_GFX1102         = 0x047,
   EF_AMDGPU_MACH_AMDGCN_GFX1200         = 0x048,
-  EF_AMDGPU_MACH_AMDGCN_RESERVED_0X49   = 0x049,
+  EF_AMDGPU_MACH_AMDGCN_GFX1250         = 0x049,
   EF_AMDGPU_MACH_AMDGCN_GFX1151         = 0x04a,
   EF_AMDGPU_MACH_AMDGCN_RESERVED_0X4B   = 0x04b,
   EF_AMDGPU_MACH_AMDGCN_GFX942          = 0x04c,
diff --git a/llvm/include/llvm/TargetParser/TargetParser.h b/llvm/include/llvm/TargetParser/TargetParser.h
index b4a92cc6b6c4..2ea7829d668a 100644
--- a/llvm/include/llvm/TargetParser/TargetParser.h
+++ b/llvm/include/llvm/TargetParser/TargetParser.h
@@ -112,9 +112,10 @@ enum GPUKind : uint32_t {
 
   GK_GFX1200 = 100,
   GK_GFX1201 = 101,
+  GK_GFX1250 = 102,
 
   GK_AMDGCN_FIRST = GK_GFX600,
-  GK_AMDGCN_LAST = GK_GFX1201,
+  GK_AMDGCN_LAST = GK_GFX1250,
 
   GK_GFX9_GENERIC = 192,
   GK_GFX10_1_GENERIC = 193,
diff --git a/llvm/lib/Object/ELFObjectFile.cpp b/llvm/lib/Object/ELFObjectFile.cpp
index dd240550cba0..cce5c74aa4d1 100644
--- a/llvm/lib/Object/ELFObjectFile.cpp
+++ b/llvm/lib/Object/ELFObjectFile.cpp
@@ -597,6 +597,8 @@ StringRef ELFObjectFileBase::getAMDGPUCPUName() const {
     return "gfx1200";
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201:
     return "gfx1201";
+  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250:
+    return "gfx1250";
 
   // Generic AMDGCN targets
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC:
diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp
index 520e956fdab9..fc485b6656e0 100644
--- a/llvm/lib/ObjectYAML/ELFYAML.cpp
+++ b/llvm/lib/ObjectYAML/ELFYAML.cpp
@@ -632,6 +632,7 @@ void ScalarBitSetTraits<ELFYAML::ELF_EF>::bitset(IO &IO,
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1153, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1200, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1201, EF_AMDGPU_MACH);
+    BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX1250, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC, EF_AMDGPU_MACH);
     BCaseMask(EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC, EF_AMDGPU_MACH);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index a9b39eebbcdc..559328459141 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -487,6 +487,12 @@ def FeatureGFX12Insts : SubtargetFeature<"gfx12-insts",
   "Additional instructions for GFX12+"
 >;
 
+def FeatureGFX1250Insts : SubtargetFeature<"gfx1250-insts",
+  "GFX1250Insts",
+  "true",
+  "Additional instructions for GFX1250+"
+>;
+
 def FeatureGFX10_3Insts : SubtargetFeature<"gfx10-3-insts",
   "GFX10_3Insts",
   "true",
@@ -1882,6 +1888,53 @@ def FeatureISAVersion12 : FeatureSet<
    FeatureBVHDualAndBVH8Insts
    ]>;
 
+def FeatureISAVersion12_50 : FeatureSet<
+  [FeatureGFX12,
+   FeatureGFX1250Insts,
+   FeatureCuMode,
+   FeatureLDSBankCount32,
+   FeatureDLInsts,
+   FeatureFmacF64Inst,
+   FeaturePackedFP32Ops,
+   FeatureDot7Insts,
+   FeatureDot8Insts,
+   FeatureWavefrontSize32,
+   FeatureShaderCyclesHiLoRegisters,
+   FeatureArchitectedFlatScratch,
+   FeatureArchitectedSGPRs,
+   FeatureAtomicFaddRtnInsts,
+   FeatureAtomicFaddNoRtnInsts,
+   FeatureAtomicDsPkAdd16Insts,
+   FeatureAtomicFlatPkAdd16Insts,
+   FeatureAtomicBufferGlobalPkAddF16Insts,
+   FeatureAtomicGlobalPkAddBF16Inst,
+   FeatureAtomicBufferPkAddBF16Inst,
+   FeatureFlatAtomicFaddF32Inst,
+   FeatureFP8ConversionInsts,
+   FeaturePackedTID,
+   FeatureVcmpxPermlaneHazard,
+   FeatureSALUFloatInsts,
+   FeaturePseudoScalarTrans,
+   FeatureHasRestrictedSOffset,
+   FeatureScalarDwordx3Loads,
+   FeatureDPPSrc1SGPR,
+   FeatureBitOp3Insts,
+   FeatureBF16ConversionInsts,
+   FeatureCvtPkF16F32Inst,
+   FeatureMinimum3Maximum3PKF16,
+   FeaturePrngInst,
+   FeaturePermlane16Swap,
+   FeatureAshrPkInsts,
+   FeatureSupportsSRAMECC,
+   FeatureMaxHardClauseLength63,
+   FeatureAtomicFMinFMaxF64GlobalInsts,
+   FeatureAtomicFMinFMaxF64FlatInsts,
+   FeatureFlatBufferGlobalAtomicFaddF64Inst,
+   FeatureMemoryAtomicFAddF32DenormalSupport,
+   FeatureKernargPreload,
+   FeatureLshlAddU64Inst,
+]>;
+
 def FeatureISAVersion12_Generic: FeatureSet<
   !listconcat(FeatureISAVersion12.Features,
     [FeatureRequiresCOV6])>;
diff --git a/llvm/lib/Target/AMDGPU/GCNProcessors.td b/llvm/lib/Target/AMDGPU/GCNProcessors.td
index 0b372e29efe6..0b331bd3f3fb 100644
--- a/llvm/lib/Target/AMDGPU/GCNProcessors.td
+++ b/llvm/lib/Target/AMDGPU/GCNProcessors.td
@@ -325,3 +325,7 @@ def : ProcessorModel<"gfx1201", GFX12SpeedModel,
 def : ProcessorModel<"gfx12-generic", GFX12SpeedModel,
   FeatureISAVersion12_Generic.Features
 >;
+
+def : ProcessorModel<"gfx1250", GFX12SpeedModel,
+  FeatureISAVersion12_50.Features
+>;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 202e5b38f0a4..dd57cc96e41c 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -113,6 +113,7 @@ protected:
   bool GFX10Insts = false;
   bool GFX11Insts = false;
   bool GFX12Insts = false;
+  bool GFX1250Insts = false;
   bool GFX10_3Insts = false;
   bool GFX7GFX8GFX9Insts = false;
   bool SGPRInitBug = false;
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index 6d69bb75f293..10f6d3382368 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -117,6 +117,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) {
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153: AK = GK_GFX1153; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200: AK = GK_GFX1200; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201: AK = GK_GFX1201; break;
+  case ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250: AK = GK_GFX1250; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC:     AK = GK_GFX9_GENERIC; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC:   AK = GK_GFX9_4_GENERIC; break;
   case ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC:  AK = GK_GFX10_1_GENERIC; break;
@@ -202,6 +203,7 @@ unsigned AMDGPUTargetStreamer::getElfMach(StringRef GPU) {
   case GK_GFX1153: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1153;
   case GK_GFX1200: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1200;
   case GK_GFX1201: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1201;
+  case GK_GFX1250: return ELF::EF_AMDGPU_MACH_AMDGCN_GFX1250;
   case GK_GFX9_GENERIC:     return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC;
   case GK_GFX9_4_GENERIC:   return ELF::EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC;
   case GK_GFX10_1_GENERIC:  return ELF::EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC;
diff --git a/llvm/lib/TargetParser/TargetParser.cpp b/llvm/lib/TargetParser/TargetParser.cpp
index 90791dfb7b7c..49442c30eb44 100644
--- a/llvm/lib/TargetParser/TargetParser.cpp
+++ b/llvm/lib/TargetParser/TargetParser.cpp
@@ -174,6 +174,7 @@ constexpr GPUInfo AMDGCNGPUs[] = {
     {{"gfx1153"},   {"gfx1153"}, GK_GFX1153, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1200"},   {"gfx1200"}, GK_GFX1200, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
     {{"gfx1201"},   {"gfx1201"}, GK_GFX1201, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_WGP},
+    {{"gfx1250"},   {"gfx1250"}, GK_GFX1250, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32},
 
     {{"gfx9-generic"},      {"gfx9-generic"},    GK_GFX9_GENERIC,    FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_XNACK},
     {{"gfx10-1-generic"},   {"gfx10-1-generic"}, GK_GFX10_1_GENERIC, FEATURE_FAST_FMA_F32|FEATURE_FAST_DENORMAL_F32|FEATURE_WAVE32|FEATURE_XNACK|FEATURE_WGP},
@@ -328,6 +329,7 @@ AMDGPU::IsaVersion AMDGPU::getIsaVersion(StringRef GPU) {
   case GK_GFX1153: return {11, 5, 3};
   case GK_GFX1200: return {12, 0, 0};
   case GK_GFX1201: return {12, 0, 1};
+  case GK_GFX1250: return {12, 5, 0};
 
   // Generic targets return the lowest common denominator
   // within their family. That is, the ISA that is the most
@@ -425,6 +427,33 @@ void AMDGPU::fillAMDGPUFeatureMap(StringRef GPU, const Triple &T,
   } else if (T.isAMDGCN()) {
     AMDGPU::GPUKind Kind = parseArchAMDGCN(GPU);
     switch (Kind) {
+    case GK_GFX1250:
+      Features["ci-insts"] = true;
+      Features["dot7-insts"] = true;
+      Features["dot8-insts"] = true;
+      Features["dl-insts"] = true;
+      Features["16-bit-insts"] = true;
+      Features["dpp"] = true;
+      Features["gfx8-insts"] = true;
+      Features["gfx9-insts"] = true;
+      Features["gfx10-insts"] = true;
+      Features["gfx10-3-insts"] = true;
+      Features["gfx11-insts"] = true;
+      Features["gfx12-insts"] = true;
+      Features["gfx1250-insts"] = true;
+      Features["bitop3-insts"] = true;
+      Features["prng-inst"] = true;
+      Features["fp8-conversion-insts"] = true;
+      Features["permlane16-swap"] = true;
+      Features["ashr-pk-insts"] = true;
+      Features["atomic-buffer-pk-add-bf16-inst"] = true;
+      Features["atomic-fadd-rtn-insts"] = true;
+      Features["atomic-buffer-global-pk-add-f16-insts"] = true;
+      Features["atomic-flat-pk-add-16-insts"] = true;
+      Features["atomic-global-pk-add-bf16-inst"] = true;
+      Features["atomic-ds-pk-add-16-insts"] = true;
+      Features["setprio-inc-wg-inst"] = true;
+      break;
     case GK_GFX1201:
     case GK_GFX1200:
     case GK_GFX12_GENERIC:
@@ -678,6 +707,7 @@ static bool isWave32Capable(StringRef GPU, const Triple &T) {
   // XXX - What does the member GPU mean if device name string passed here?
   if (T.isAMDGCN()) {
     switch (parseArchAMDGCN(GPU)) {
+    case GK_GFX1250:
     case GK_GFX1201:
     case GK_GFX1200:
     case GK_GFX1153:
diff --git a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
index c954e1fe124e..9940cc53e612 100644
--- a/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
+++ b/llvm/test/CodeGen/AMDGPU/directive-amdgcn-target.ll
@@ -106,6 +106,7 @@
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1153 < %s | FileCheck --check-prefixes=GFX1153 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck --check-prefixes=GFX1200 %s
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1201 < %s | FileCheck --check-prefixes=GFX1201 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 < %s | FileCheck --check-prefixes=GFX1250 %s
 
 ; RUN: llc --amdhsa-code-object-version=6 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx9-generic -mattr=-xnack < %s | FileCheck --check-prefixes=GFX9_GENERIC_NOXNACK %s
 ; RUN: llc --amdhsa-code-object-version=6 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx9-generic -mattr=+xnack < %s | FileCheck --check-prefixes=GFX9_GENERIC_XNACK %s
@@ -203,6 +204,7 @@
 ; GFX1153: .amdgcn_target "amdgcn-amd-amdhsa--gfx1153"
 ; GFX1200: .amdgcn_target "amdgcn-amd-amdhsa--gfx1200"
 ; GFX1201: .amdgcn_target "amdgcn-amd-amdhsa--gfx1201"
+; GFX1250: .amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
 
 ; GFX9_GENERIC_NOXNACK:     .amdgcn_target "amdgcn-amd-amdhsa--gfx9-generic:xnack-"
 ; GFX9_GENERIC_XNACK:       .amdgcn_target "amdgcn-amd-amdhsa--gfx9-generic:xnack+"
diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
index 65039b471694..32cacf0d6dd9 100644
--- a/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
+++ b/llvm/test/CodeGen/AMDGPU/elf-header-flags-mach.ll
@@ -77,6 +77,7 @@
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1153 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1153 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1200 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1200 %s
 ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1201 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1201 %s
+; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx1250 < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX1250 %s
 
 ; RUN: llc -filetype=obj --amdhsa-code-object-version=6 -mtriple=amdgcn -mcpu=gfx9-generic < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX9_GENERIC %s
 ; RUN: llc -filetype=obj --amdhsa-code-object-version=6 -mtriple=amdgcn -mcpu=gfx9-4-generic < %s | llvm-readobj --file-header - | FileCheck --check-prefixes=ALL,ARCH-GCN,GFX9_4_GENERIC %s
@@ -158,6 +159,7 @@
 ; GFX1153:       EF_AMDGPU_MACH_AMDGCN_GFX1153 (0x58)
 ; GFX1200:       EF_AMDGPU_MACH_AMDGCN_GFX1200 (0x48)
 ; GFX1201:       EF_AMDGPU_MACH_AMDGCN_GFX1201 (0x4E)
+; GFX1250:       EF_AMDGPU_MACH_AMDGCN_GFX1250 (0x49)
 
 ; GFX9_GENERIC:       EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC (0x51)
 ; GFX9_4_GENERIC:     EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC (0x5F)
diff --git a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
index 8d5ce006e24d..51590819f8b9 100644
--- a/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
+++ b/llvm/test/Object/AMDGPU/elf-header-flags-mach.yaml
@@ -242,6 +242,10 @@
 # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1201 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1201 %s
 # RUN: obj2yaml %t.o.AMDGCN_GFX1201 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1201 %s
 
+# RUN: sed -e 's/<BITS>/64/' -e 's/<MACH>/AMDGCN_GFX1250/' %s | yaml2obj -o %t.o.AMDGCN_GFX1250
+# RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX1250 | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX1250 %s
+# RUN: obj2yaml %t.o.AMDGCN_GFX1250 | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX1250 %s
+
 # RUN: sed -e 's/<BITS>/64/' -e 's/<MACH>/AMDGCN_GFX9_GENERIC/' %s | yaml2obj -o %t.o.AMDGCN_GFX9_GENERIC
 # RUN: llvm-readobj -S --file-headers %t.o.AMDGCN_GFX9_GENERIC | FileCheck --check-prefixes=ELF-AMDGCN-ALL,ELF-AMDGCN-GFX9_GENERIC %s
 # RUN: obj2yaml %t.o.AMDGCN_GFX9_GENERIC | FileCheck --check-prefixes=YAML-AMDGCN-ALL,YAML-AMDGCN-GFX9_GENERIC %s
@@ -467,6 +471,9 @@
 # ELF-AMDGCN-GFX1201:   EF_AMDGPU_MACH_AMDGCN_GFX1201 (0x4E)
 # YAML-AMDGCN-GFX1201:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1201 ]
 
+# ELF-AMDGCN-GFX1250:   EF_AMDGPU_MACH_AMDGCN_GFX1250 (0x49)
+# YAML-AMDGCN-GFX1250:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX1250 ]
+
 # ELF-AMDGCN-GFX9_GENERIC:   EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC (0x51)
 # YAML-AMDGCN-GFX9_GENERIC:  Flags: [ EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC ]
 
diff --git a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
index 0e392e228715..155ced379b0c 100644
--- a/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
+++ b/llvm/test/tools/llvm-objdump/ELF/AMDGPU/subtarget.ll
@@ -21,6 +21,11 @@ define amdgpu_kernel void @test_kernel() {
 ; RUN: llvm-objdump -D %t.o > %t-detect.txt
 ; RUN: diff %t-specify.txt %t-detect.txt
 
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1250 -filetype=obj -O0 -o %t.o %s
+; RUN: llvm-objdump -D --arch-name=amdgcn --mcpu=gfx1250 %t.o > %t-specify.txt
+; RUN: llvm-objdump -D %t.o > %t-detect.txt
+; RUN: diff %t-specify.txt %t-detect.txt
+
 ; ----------------------------------GFX11--------------------------------------
 ;
 ; RUN: llc -mtriple=amdgcn-amd-amdhsa --amdhsa-code-object-version=6 -mcpu=gfx11-generic -filetype=obj -O0 -o %t.o %s
diff --git a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
index dd9eaaeabd8c..8edb887ab005 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
+++ b/llvm/test/tools/llvm-readobj/ELF/AMDGPU/elf-headers.test
@@ -454,6 +454,15 @@
 # RUN: yaml2obj %s -o %t -DABI_VERSION=4 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC
 # RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=4 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX12_GENERIC -DFLAG_VALUE=0x59
 
+# RUN: yaml2obj %s -o %t -DABI_VERSION=0 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=0 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250 -DFLAG_VALUE=0x49
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=1 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=1 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250 -DFLAG_VALUE=0x49
+
+# RUN: yaml2obj %s -o %t -DABI_VERSION=2 -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250
+# RUN: llvm-readobj -h %t | FileCheck %s --check-prefixes=ALL,KNOWN-ABI-VERSION,SINGLE-FLAG --match-full-lines -DABI_VERSION=2 -DFILE=%t -DFLAG_NAME=EF_AMDGPU_MACH_AMDGCN_GFX1250 -DFLAG_VALUE=0x49
+
 --- !ELF
 FileHeader:
   Class:           ELFCLASS64
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index abaf6077ba9e..7be61dcce841 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -1640,6 +1640,7 @@ const EnumEntry<unsigned> ElfHeaderMipsFlags[] = {
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1153, "gfx1153"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1200, "gfx1200"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1201, "gfx1201"),                          \
+  ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX1250, "gfx1250"),                          \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX9_GENERIC, "gfx9-generic"),                \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX9_4_GENERIC, "gfx9-4-generic"),            \
   ENUM_ENT(EF_AMDGPU_MACH_AMDGCN_GFX10_1_GENERIC, "gfx10-1-generic"),          \
-- 
cgit v1.2.3


From b8d3efa189620bfd48dab5fb05b923560fb1e2d5 Mon Sep 17 00:00:00 2001
From: Maksim Panchenko <maks@fb.com>
Date: Thu, 19 Jun 2025 23:09:34 -0700
Subject: [BOLT][Linux] Fix linux_banner lookup (#144962)

While detecting the Linux kernel version, look for `linux_banner` symbol
with local visibility if the global one was not found.

Fixes #144847
---
 bolt/lib/Rewrite/LinuxKernelRewriter.cpp | 42 +++++++++++++++++++-------------
 bolt/test/X86/linux-version.S            | 11 +++++++++
 2 files changed, 36 insertions(+), 17 deletions(-)

diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 5a5e044184d0..174721a3a053 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -432,25 +432,33 @@ public:
 };
 
 Error LinuxKernelRewriter::detectLinuxKernelVersion() {
-  if (BinaryData *BD = BC.getBinaryDataByName("linux_banner")) {
-    const BinarySection &Section = BD->getSection();
-    const std::string S =
-        Section.getContents().substr(BD->getOffset(), BD->getSize()).str();
-
-    const std::regex Re(R"---(Linux version ((\d+)\.(\d+)(\.(\d+))?))---");
-    std::smatch Match;
-    if (std::regex_search(S, Match, Re)) {
-      const unsigned Major = std::stoi(Match[2].str());
-      const unsigned Minor = std::stoi(Match[3].str());
-      const unsigned Rev = Match[5].matched ? std::stoi(Match[5].str()) : 0;
-      LinuxKernelVersion = LKVersion(Major, Minor, Rev);
-      BC.outs() << "BOLT-INFO: Linux kernel version is " << Match[1].str()
-                << "\n";
-      return Error::success();
-    }
+  // Check for global and local linux_banner symbol.
+  BinaryData *BD = BC.getBinaryDataByName("linux_banner");
+  if (!BD)
+    BD = BC.getBinaryDataByName("linux_banner/1");
+
+  if (!BD)
+    return createStringError(errc::executable_format_error,
+                             "unable to locate linux_banner");
+
+  const BinarySection &Section = BD->getSection();
+  const std::string S =
+      Section.getContents().substr(BD->getOffset(), BD->getSize()).str();
+
+  const std::regex Re(R"---(Linux version ((\d+)\.(\d+)(\.(\d+))?))---");
+  std::smatch Match;
+  if (std::regex_search(S, Match, Re)) {
+    const unsigned Major = std::stoi(Match[2].str());
+    const unsigned Minor = std::stoi(Match[3].str());
+    const unsigned Rev = Match[5].matched ? std::stoi(Match[5].str()) : 0;
+    LinuxKernelVersion = LKVersion(Major, Minor, Rev);
+    BC.outs() << "BOLT-INFO: Linux kernel version is " << Match[1].str()
+              << "\n";
+    return Error::success();
   }
+
   return createStringError(errc::executable_format_error,
-                           "Linux kernel version is unknown");
+                           "Linux kernel version is unknown: " + S);
 }
 
 void LinuxKernelRewriter::processLKSections() {
diff --git a/bolt/test/X86/linux-version.S b/bolt/test/X86/linux-version.S
index e680d0d64a21..a3d7f365304a 100644
--- a/bolt/test/X86/linux-version.S
+++ b/bolt/test/X86/linux-version.S
@@ -17,6 +17,11 @@
 # RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr
 # RUN: llvm-bolt %t.exe -o %t.out 2>&1 | FileCheck --check-prefix=CHECK-C %s
 
+# RUN: %clang -DD -target x86_64-unknown-unknown \
+# RUN:   %cflags -nostdlib %s -o %t.exe \
+# RUN:   -Wl,--image-base=0xffffffff80000000,--no-dynamic-linker,--no-eh-frame-hdr
+# RUN: llvm-bolt %t.exe -o %t.out 2>&1 | FileCheck --check-prefix=CHECK-D %s
+
   .text
   .globl foo
   .type foo, %function
@@ -46,6 +51,12 @@ linux_banner:
 #endif
 # CHECK-C: BOLT-INFO: Linux kernel version is 6.6
 
+#ifdef D
+  .hidden linux_banner
+  .string  "Linux version 6.6.15.2-2-xxx\n"
+#endif
+# CHECK-D: BOLT-INFO: Linux kernel version is 6.6
+
   .size  linux_banner, . - linux_banner
 
 ## Fake Linux Kernel sections.
-- 
cgit v1.2.3


From 7cbb1411550ef6caab18a9360f1549d6029ffe86 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Thu, 19 Jun 2025 23:29:50 -0700
Subject: [clang] Migrate away from ArrayRef(std::nullopt) (NFC) (#144982)

ArrayRef has a constructor that accepts std::nullopt.  This
constructor dates back to the days when we still had llvm::Optional.

Since the use of std::nullopt outside the context of std::optional is
kind of abuse and not intuitive to new comers, I would like to move
away from the constructor and eventually remove it.

This patch takes care of the clang side of the migration.
---
 clang/include/clang/AST/TypeProperties.td | 2 +-
 clang/lib/AST/ASTContext.cpp              | 4 ++--
 clang/lib/AST/ASTDiagnostic.cpp           | 4 ++--
 clang/lib/AST/ASTImporter.cpp             | 2 +-
 clang/lib/AST/DeclTemplate.cpp            | 2 +-
 clang/lib/AST/QualTypeNames.cpp           | 5 ++---
 clang/lib/Basic/Targets/Xtensa.h          | 2 +-
 clang/lib/CodeGen/CGBuiltin.cpp           | 2 +-
 clang/lib/Driver/ToolChains/Clang.cpp     | 4 ++--
 clang/lib/Sema/SemaConcept.cpp            | 4 ++--
 clang/lib/Sema/SemaExpr.cpp               | 2 +-
 clang/lib/Sema/TreeTransform.h            | 2 +-
 12 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/clang/include/clang/AST/TypeProperties.td b/clang/include/clang/AST/TypeProperties.td
index 6e44bce893e7..d7dbf1b43df2 100644
--- a/clang/include/clang/AST/TypeProperties.td
+++ b/clang/include/clang/AST/TypeProperties.td
@@ -753,7 +753,7 @@ let Class = TemplateSpecializationType in {
   }
 
   def : Creator<[{
-    return ctx.getTemplateSpecializationType(templateName, args, std::nullopt, UnderlyingType);
+    return ctx.getTemplateSpecializationType(templateName, args, {}, UnderlyingType);
   }]>;
 }
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 189e67e4eed0..74be2871f270 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -14283,7 +14283,7 @@ static QualType getCommonNonSugarTypeNode(ASTContext &Ctx, const Type *X,
         ::getCommonTemplateNameChecked(Ctx, TX->getTemplateName(),
                                        TY->getTemplateName(),
                                        /*IgnoreDeduced=*/true),
-        As, /*CanonicalArgs=*/std::nullopt, X->getCanonicalTypeInternal());
+        As, /*CanonicalArgs=*/{}, X->getCanonicalTypeInternal());
   }
   case Type::Decltype: {
     const auto *DX = cast<DecltypeType>(X);
@@ -14529,7 +14529,7 @@ static QualType getCommonSugarTypeNode(ASTContext &Ctx, const Type *X,
                                    TY->template_arguments()))
       return QualType();
     return Ctx.getTemplateSpecializationType(CTN, As,
-                                             /*CanonicalArgs=*/std::nullopt,
+                                             /*CanonicalArgs=*/{},
                                              Ctx.getQualifiedType(Underlying));
   }
   case Type::Typedef: {
diff --git a/clang/lib/AST/ASTDiagnostic.cpp b/clang/lib/AST/ASTDiagnostic.cpp
index a00d5801f054..522abd5912db 100644
--- a/clang/lib/AST/ASTDiagnostic.cpp
+++ b/clang/lib/AST/ASTDiagnostic.cpp
@@ -130,7 +130,7 @@ QualType clang::desugarForDiagnostic(ASTContext &Context, QualType QT,
         if (DesugarArgument) {
           ShouldAKA = true;
           QT = Context.getTemplateSpecializationType(
-              TST->getTemplateName(), Args, /*CanonicalArgs=*/std::nullopt, QT);
+              TST->getTemplateName(), Args, /*CanonicalArgs=*/{}, QT);
         }
         break;
       }
@@ -1143,7 +1143,7 @@ class TemplateDiff {
 
     Ty = Context.getTemplateSpecializationType(
         TemplateName(CTSD->getSpecializedTemplate()),
-        CTSD->getTemplateArgs().asArray(), /*CanonicalArgs=*/std::nullopt,
+        CTSD->getTemplateArgs().asArray(), /*CanonicalArgs=*/{},
         Ty.getLocalUnqualifiedType().getCanonicalType());
 
     return Ty->getAs<TemplateSpecializationType>();
diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp
index 96a5e2eeaa4d..4621ebb854d8 100644
--- a/clang/lib/AST/ASTImporter.cpp
+++ b/clang/lib/AST/ASTImporter.cpp
@@ -1664,7 +1664,7 @@ ExpectedType ASTNodeImporter::VisitTemplateSpecializationType(
   if (!ToUnderlyingOrErr)
     return ToUnderlyingOrErr.takeError();
   return Importer.getToContext().getTemplateSpecializationType(
-      *ToTemplateOrErr, ToTemplateArgs, std::nullopt, *ToUnderlyingOrErr);
+      *ToTemplateOrErr, ToTemplateArgs, {}, *ToUnderlyingOrErr);
 }
 
 ExpectedType ASTNodeImporter::VisitElaboratedType(const ElaboratedType *T) {
diff --git a/clang/lib/AST/DeclTemplate.cpp b/clang/lib/AST/DeclTemplate.cpp
index e1ef2188dbdb..5035f2d33b0a 100644
--- a/clang/lib/AST/DeclTemplate.cpp
+++ b/clang/lib/AST/DeclTemplate.cpp
@@ -669,7 +669,7 @@ ClassTemplateDecl::getInjectedClassNameSpecialization() {
   CommonPtr->InjectedClassNameType =
       Context.getTemplateSpecializationType(Name,
                                             /*SpecifiedArgs=*/TemplateArgs,
-                                            /*CanonicalArgs=*/std::nullopt);
+                                            /*CanonicalArgs=*/{});
   return CommonPtr->InjectedClassNameType;
 }
 
diff --git a/clang/lib/AST/QualTypeNames.cpp b/clang/lib/AST/QualTypeNames.cpp
index 4d11a3b62331..39703d6d7b88 100644
--- a/clang/lib/AST/QualTypeNames.cpp
+++ b/clang/lib/AST/QualTypeNames.cpp
@@ -140,7 +140,7 @@ static const Type *getFullyQualifiedTemplateType(const ASTContext &Ctx,
     if (MightHaveChanged) {
       QualType QT = Ctx.getTemplateSpecializationType(
           TST->getTemplateName(), FQArgs,
-          /*CanonicalArgs=*/std::nullopt, TST->desugar());
+          /*CanonicalArgs=*/{}, TST->desugar());
       // getTemplateSpecializationType returns a fully qualified
       // version of the specialization itself, so no need to qualify
       // it.
@@ -172,8 +172,7 @@ static const Type *getFullyQualifiedTemplateType(const ASTContext &Ctx,
         TemplateName TN(TSTDecl->getSpecializedTemplate());
         QualType QT = Ctx.getTemplateSpecializationType(
             TN, FQArgs,
-            /*CanonicalArgs=*/std::nullopt,
-            TSTRecord->getCanonicalTypeInternal());
+            /*CanonicalArgs=*/{}, TSTRecord->getCanonicalTypeInternal());
         // getTemplateSpecializationType returns a fully qualified
         // version of the specialization itself, so no need to qualify
         // it.
diff --git a/clang/lib/Basic/Targets/Xtensa.h b/clang/lib/Basic/Targets/Xtensa.h
index 470835aacff5..f3558ac247be 100644
--- a/clang/lib/Basic/Targets/Xtensa.h
+++ b/clang/lib/Basic/Targets/Xtensa.h
@@ -77,7 +77,7 @@ public:
   }
 
   ArrayRef<TargetInfo::GCCRegAlias> getGCCRegAliases() const override {
-    return std::nullopt;
+    return {};
   }
 
   bool validateAsmConstraint(const char *&Name,
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 1f6927435167..2c011a951986 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -2032,7 +2032,7 @@ Value *CodeGenFunction::EmitCheckedArgForAssume(const Expr *E) {
       std::make_pair(ArgValue, CheckOrdinal), CheckHandler,
       {EmitCheckSourceLocation(E->getExprLoc()),
        llvm::ConstantInt::get(Builder.getInt8Ty(), BCK_AssumePassedFalse)},
-      std::nullopt);
+      {});
   return ArgValue;
 }
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index e910a2bedeeb..2bb42a319ecc 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -8953,7 +8953,7 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA,
   C.addCommand(std::make_unique<Command>(
       JA, *this, ResponseFileSupport::None(),
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
-      CmdArgs, std::nullopt, Output));
+      CmdArgs, ArrayRef<InputInfo>(), Output));
 }
 
 void OffloadBundler::ConstructJobMultipleOutputs(
@@ -9040,7 +9040,7 @@ void OffloadBundler::ConstructJobMultipleOutputs(
   C.addCommand(std::make_unique<Command>(
       JA, *this, ResponseFileSupport::None(),
       TCArgs.MakeArgString(getToolChain().GetProgramPath(getShortName())),
-      CmdArgs, std::nullopt, Outputs));
+      CmdArgs, ArrayRef<InputInfo>(), Outputs));
 }
 
 void OffloadPackager::ConstructJob(Compilation &C, const JobAction &JA,
diff --git a/clang/lib/Sema/SemaConcept.cpp b/clang/lib/Sema/SemaConcept.cpp
index a4f660c39b4b..1594b4423e4d 100644
--- a/clang/lib/Sema/SemaConcept.cpp
+++ b/clang/lib/Sema/SemaConcept.cpp
@@ -1081,10 +1081,10 @@ static bool CheckFunctionConstraintsWithoutInstantiation(
   // FIXME: Add TemplateArgs through the 'Innermost' parameter once
   // the refactoring of getTemplateInstantiationArgs() relands.
   MultiLevelTemplateArgumentList MLTAL;
-  MLTAL.addOuterTemplateArguments(Template, std::nullopt, /*Final=*/false);
+  MLTAL.addOuterTemplateArguments(Template, {}, /*Final=*/false);
   SemaRef.getTemplateInstantiationArgs(
       MLTAL, /*D=*/FD, FD,
-      /*Final=*/false, /*Innermost=*/std::nullopt, /*RelativeToPrimary=*/true,
+      /*Final=*/false, /*Innermost=*/{}, /*RelativeToPrimary=*/true,
       /*Pattern=*/nullptr, /*ForConstraintInstantiation=*/true);
   MLTAL.replaceInnermostTemplateArguments(Template, TemplateArgs);
 
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index ebc43157d4c2..fc2819458a4f 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -21212,7 +21212,7 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
     }
     if (TST.isNull())
       TST = Context.getTemplateSpecializationType(
-          TN, ULE->template_arguments(), /*CanonicalArgs=*/std::nullopt,
+          TN, ULE->template_arguments(), /*CanonicalArgs=*/{},
           HasAnyDependentTA ? Context.DependentTy : Context.IntTy);
     QualType ET =
         Context.getElaboratedType(ElaboratedTypeKeyword::None, NNS, TST);
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3e33fb73e01b..26bee7a96de2 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -16222,7 +16222,7 @@ TreeTransform<Derived>::TransformSizeOfPackExpr(SizeOfPackExpr *E) {
   return getDerived().RebuildSizeOfPackExpr(
       E->getOperatorLoc(), E->getPack(), E->getPackLoc(), E->getRParenLoc(),
       /*Length=*/static_cast<unsigned>(Args.size()),
-      /*PartialArgs=*/std::nullopt);
+      /*PartialArgs=*/{});
 }
 
 template <typename Derived>
-- 
cgit v1.2.3


From 0f302f38b0014a0018031ffb3cb898fdc7d90880 Mon Sep 17 00:00:00 2001
From: no92 <no92@users.noreply.github.com>
Date: Fri, 20 Jun 2025 08:40:20 +0200
Subject: [clang] Add managarm support (#144791)

This is a repost of the quickly reverted #139271. The failing buildbot
tests have been fixed and pass on my machine now.
---
 clang/lib/Basic/Targets.cpp                        |   9 +
 clang/lib/Basic/Targets/OSTargets.h                |  30 +++
 clang/lib/Driver/CMakeLists.txt                    |   1 +
 clang/lib/Driver/Driver.cpp                        |   4 +
 clang/lib/Driver/ToolChains/Gnu.cpp                |   2 +
 clang/lib/Driver/ToolChains/Managarm.cpp           | 218 +++++++++++++++++
 clang/lib/Driver/ToolChains/Managarm.h             |  55 +++++
 clang/lib/Lex/InitHeaderSearch.cpp                 |   1 +
 .../lib/aarch64-managarm-mlibc/.keep               |   0
 .../lib/riscv64-managarm-mlibc/.keep               |   0
 .../lib/x86_64-managarm-mlibc/.keep                |   0
 .../lib64/aarch64-managarm-mlibc/.keep             |   0
 .../lib64/riscv64-managarm-mlibc/.keep             |   0
 .../lib64/x86_64-managarm-mlibc/.keep              |   0
 .../include/aarch64-managarm-mlibc/c++/10/.keep    |   0
 .../basic_managarm_tree/usr/include/c++/10/.keep   |   0
 .../basic_managarm_tree/usr/include/c++/v1/.keep   |   0
 .../include/riscv64-managarm-mlibc/c++/10/.keep    |   0
 .../usr/include/x86_64-managarm-mlibc/c++/10/.keep |   0
 .../usr/lib/aarch64-managarm-mlibc/.keep           |   0
 .../lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o   |   0
 .../lib/gcc/aarch64-managarm-mlibc/10/crtbeginS.o  |   0
 .../lib/gcc/aarch64-managarm-mlibc/10/crtbeginT.o  |   0
 .../lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o   |   0
 .../lib/gcc/riscv64-managarm-mlibc/10/crtbeginS.o  |   0
 .../lib/gcc/riscv64-managarm-mlibc/10/crtbeginT.o  |   0
 .../lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o    |   0
 .../lib/gcc/x86_64-managarm-mlibc/10/crtbeginS.o   |   0
 .../lib/gcc/x86_64-managarm-mlibc/10/crtbeginT.o   |   0
 .../usr/lib/riscv64-managarm-mlibc/.keep           |   0
 .../usr/lib/x86_64-managarm-mlibc/.keep            |   0
 .../Inputs/basic_managarm_tree/usr/lib64/.keep     |   0
 clang/test/Driver/managarm.cpp                     | 267 +++++++++++++++++++++
 clang/test/Preprocessor/init.c                     |   5 +
 .../Preprocessor/predefined-macros-no-warnings.c   |   3 +
 35 files changed, 595 insertions(+)
 create mode 100644 clang/lib/Driver/ToolChains/Managarm.cpp
 create mode 100644 clang/lib/Driver/ToolChains/Managarm.h
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib/aarch64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib/riscv64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib/x86_64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib64/aarch64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib64/riscv64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/lib64/x86_64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/include/aarch64-managarm-mlibc/c++/10/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/10/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/v1/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/include/riscv64-managarm-mlibc/c++/10/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/include/x86_64-managarm-mlibc/c++/10/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/aarch64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginS.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginT.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginS.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginT.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginS.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginT.o
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/riscv64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/x86_64-managarm-mlibc/.keep
 create mode 100644 clang/test/Driver/Inputs/basic_managarm_tree/usr/lib64/.keep
 create mode 100644 clang/test/Driver/managarm.cpp

diff --git a/clang/lib/Basic/Targets.cpp b/clang/lib/Basic/Targets.cpp
index 9889141ad208..af1111a86330 100644
--- a/clang/lib/Basic/Targets.cpp
+++ b/clang/lib/Basic/Targets.cpp
@@ -164,6 +164,9 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple,
         return std::make_unique<OHOSTargetInfo<AArch64leTargetInfo>>(Triple,
                                                                      Opts);
       }
+    case llvm::Triple::Managarm:
+      return std::make_unique<ManagarmTargetInfo<AArch64leTargetInfo>>(Triple,
+                                                                       Opts);
     case llvm::Triple::NetBSD:
       return std::make_unique<NetBSDTargetInfo<AArch64leTargetInfo>>(Triple,
                                                                      Opts);
@@ -466,6 +469,9 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple,
         return std::make_unique<OHOSTargetInfo<RISCV64TargetInfo>>(Triple,
                                                                    Opts);
       }
+    case llvm::Triple::Managarm:
+      return std::make_unique<ManagarmTargetInfo<RISCV64TargetInfo>>(Triple,
+                                                                     Opts);
     default:
       return std::make_unique<RISCV64TargetInfo>(Triple, Opts);
     }
@@ -654,6 +660,9 @@ std::unique_ptr<TargetInfo> AllocateTarget(const llvm::Triple &Triple,
       return std::make_unique<PS5OSTargetInfo<X86_64TargetInfo>>(Triple, Opts);
     case llvm::Triple::Hurd:
       return std::make_unique<HurdTargetInfo<X86_64TargetInfo>>(Triple, Opts);
+    case llvm::Triple::Managarm:
+      return std::make_unique<ManagarmTargetInfo<X86_64TargetInfo>>(Triple,
+                                                                    Opts);
     default:
       return std::make_unique<X86_64TargetInfo>(Triple, Opts);
     }
diff --git a/clang/lib/Basic/Targets/OSTargets.h b/clang/lib/Basic/Targets/OSTargets.h
index d148b38d03c7..5dac699c2bb4 100644
--- a/clang/lib/Basic/Targets/OSTargets.h
+++ b/clang/lib/Basic/Targets/OSTargets.h
@@ -395,6 +395,36 @@ public:
   }
 };
 
+// Managarm Target
+template <typename Target>
+class LLVM_LIBRARY_VISIBILITY ManagarmTargetInfo : public OSTargetInfo<Target> {
+protected:
+  void getOSDefines(const LangOptions &Opts, const llvm::Triple &Triple,
+                    MacroBuilder &Builder) const override {
+    DefineStd(Builder, "unix", Opts);
+    Builder.defineMacro("__managarm__");
+    if (Opts.POSIXThreads)
+      Builder.defineMacro("_REENTRANT");
+    if (Opts.CPlusPlus)
+      Builder.defineMacro("_GNU_SOURCE");
+    if (this->HasFloat128)
+      Builder.defineMacro("__FLOAT128__");
+  }
+
+public:
+  ManagarmTargetInfo(const llvm::Triple &Triple, const TargetOptions &Opts)
+      : OSTargetInfo<Target>(Triple, Opts) {
+    switch (Triple.getArch()) {
+    default:
+      break;
+    case llvm::Triple::x86:
+    case llvm::Triple::x86_64:
+      this->HasFloat128 = true;
+      break;
+    }
+  }
+};
+
 // NetBSD Target
 template <typename Target>
 class LLVM_LIBRARY_VISIBILITY NetBSDTargetInfo : public OSTargetInfo<Target> {
diff --git a/clang/lib/Driver/CMakeLists.txt b/clang/lib/Driver/CMakeLists.txt
index 44e16edfb1cc..3cfd671e9d8f 100644
--- a/clang/lib/Driver/CMakeLists.txt
+++ b/clang/lib/Driver/CMakeLists.txt
@@ -66,6 +66,7 @@ add_clang_library(clangDriver
   ToolChains/HLSL.cpp
   ToolChains/Hurd.cpp
   ToolChains/Linux.cpp
+  ToolChains/Managarm.cpp
   ToolChains/MipsLinux.cpp
   ToolChains/MinGW.cpp
   ToolChains/MSP430.cpp
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index 061d60c62903..2d055ffa17a8 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -33,6 +33,7 @@
 #include "ToolChains/Linux.h"
 #include "ToolChains/MSP430.h"
 #include "ToolChains/MSVC.h"
+#include "ToolChains/Managarm.h"
 #include "ToolChains/MinGW.h"
 #include "ToolChains/MipsLinux.h"
 #include "ToolChains/NaCl.h"
@@ -6850,6 +6851,9 @@ const ToolChain &Driver::getToolChain(const ArgList &Args,
     case llvm::Triple::Fuchsia:
       TC = std::make_unique<toolchains::Fuchsia>(*this, Target, Args);
       break;
+    case llvm::Triple::Managarm:
+      TC = std::make_unique<toolchains::Managarm>(*this, Target, Args);
+      break;
     case llvm::Triple::Solaris:
       TC = std::make_unique<toolchains::Solaris>(*this, Target, Args);
       break;
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index 9203bbc91b0b..afce4fffe1d5 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -226,6 +226,8 @@ static const char *getLDMOption(const llvm::Triple &T, const ArgList &Args) {
       return "elf_iamcu";
     return "elf_i386";
   case llvm::Triple::aarch64:
+    if (T.isOSManagarm())
+      return "aarch64managarm";
     return "aarch64linux";
   case llvm::Triple::aarch64_be:
     return "aarch64linuxb";
diff --git a/clang/lib/Driver/ToolChains/Managarm.cpp b/clang/lib/Driver/ToolChains/Managarm.cpp
new file mode 100644
index 000000000000..ff455f2c6ec7
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/Managarm.cpp
@@ -0,0 +1,218 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Managarm.h"
+#include "Arch/ARM.h"
+#include "Arch/RISCV.h"
+#include "clang/Config/config.h"
+#include "clang/Driver/CommonArgs.h"
+#include "clang/Driver/Driver.h"
+#include "clang/Driver/Options.h"
+#include "clang/Driver/SanitizerArgs.h"
+#include "llvm/Option/ArgList.h"
+#include "llvm/Support/Path.h"
+
+using namespace clang::driver;
+using namespace clang::driver::toolchains;
+using namespace clang;
+using namespace llvm::opt;
+
+using tools::addPathIfExists;
+
+std::string Managarm::getMultiarchTriple(const Driver &D,
+                                         const llvm::Triple &TargetTriple,
+                                         StringRef SysRoot) const {
+  switch (TargetTriple.getArch()) {
+  default:
+    return TargetTriple.str();
+  case llvm::Triple::x86_64:
+    return "x86_64-managarm-" + TargetTriple.getEnvironmentName().str();
+  case llvm::Triple::aarch64:
+    return "aarch64-managarm-" + TargetTriple.getEnvironmentName().str();
+  case llvm::Triple::riscv64:
+    return "riscv64-managarm-" + TargetTriple.getEnvironmentName().str();
+  }
+}
+
+static StringRef getOSLibDir(const llvm::Triple &Triple, const ArgList &Args) {
+  // It happens that only x86, PPC and SPARC use the 'lib32' variant of
+  // oslibdir, and using that variant while targeting other architectures causes
+  // problems because the libraries are laid out in shared system roots that
+  // can't cope with a 'lib32' library search path being considered. So we only
+  // enable them when we know we may need it.
+  //
+  // FIXME: This is a bit of a hack. We should really unify this code for
+  // reasoning about oslibdir spellings with the lib dir spellings in the
+  // GCCInstallationDetector, but that is a more significant refactoring.
+  if (Triple.getArch() == llvm::Triple::x86 || Triple.isPPC32() ||
+      Triple.getArch() == llvm::Triple::sparc)
+    return "lib32";
+
+  if (Triple.getArch() == llvm::Triple::x86_64 && Triple.isX32())
+    return "libx32";
+
+  if (Triple.getArch() == llvm::Triple::riscv32)
+    return "lib32";
+
+  return Triple.isArch32Bit() ? "lib" : "lib64";
+}
+
+Managarm::Managarm(const Driver &D, const llvm::Triple &Triple,
+                   const ArgList &Args)
+    : Generic_ELF(D, Triple, Args) {
+  GCCInstallation.init(Triple, Args);
+  Multilibs = GCCInstallation.getMultilibs();
+  SelectedMultilibs.assign({GCCInstallation.getMultilib()});
+  std::string SysRoot = computeSysRoot();
+
+  ToolChain::path_list &PPaths = getProgramPaths();
+
+  Generic_GCC::PushPPaths(PPaths);
+
+#ifdef ENABLE_LINKER_BUILD_ID
+  ExtraOpts.push_back("--build-id");
+#endif
+
+  // The selection of paths to try here is designed to match the patterns which
+  // the GCC driver itself uses, as this is part of the GCC-compatible driver.
+  // This was determined by running GCC in a fake filesystem, creating all
+  // possible permutations of these directories, and seeing which ones it added
+  // to the link paths.
+  path_list &Paths = getFilePaths();
+
+  const std::string OSLibDir = std::string(getOSLibDir(Triple, Args));
+  const std::string MultiarchTriple = getMultiarchTriple(D, Triple, SysRoot);
+
+  Generic_GCC::AddMultilibPaths(D, SysRoot, OSLibDir, MultiarchTriple, Paths);
+
+  addPathIfExists(D, concat(SysRoot, "/lib", MultiarchTriple), Paths);
+  addPathIfExists(D, concat(SysRoot, "/lib/..", OSLibDir), Paths);
+  addPathIfExists(D, concat(SysRoot, "/usr/lib", MultiarchTriple), Paths);
+  addPathIfExists(D, concat(SysRoot, "/usr", OSLibDir), Paths);
+
+  Generic_GCC::AddMultiarchPaths(D, SysRoot, OSLibDir, Paths);
+
+  addPathIfExists(D, concat(SysRoot, "/lib"), Paths);
+  addPathIfExists(D, concat(SysRoot, "/usr/lib"), Paths);
+}
+
+bool Managarm::HasNativeLLVMSupport() const { return true; }
+
+Tool *Managarm::buildLinker() const {
+  return new tools::gnutools::Linker(*this);
+}
+
+Tool *Managarm::buildAssembler() const {
+  return new tools::gnutools::Assembler(*this);
+}
+
+std::string Managarm::computeSysRoot() const {
+  if (!getDriver().SysRoot.empty())
+    return getDriver().SysRoot;
+  return std::string();
+}
+
+std::string Managarm::getDynamicLinker(const ArgList &Args) const {
+  switch (getTriple().getArch()) {
+  case llvm::Triple::aarch64:
+    return "/lib/aarch64-managarm/ld.so";
+  case llvm::Triple::riscv64: {
+    StringRef ABIName = tools::riscv::getRISCVABI(Args, getTriple());
+    return ("/lib/riscv64-managarm/ld-riscv64-" + ABIName + ".so").str();
+  }
+  case llvm::Triple::x86_64:
+    return "/lib/x86_64-managarm/ld.so";
+  default:
+    llvm_unreachable("unsupported architecture");
+  }
+}
+
+void Managarm::AddClangSystemIncludeArgs(const ArgList &DriverArgs,
+                                         ArgStringList &CC1Args) const {
+  const Driver &D = getDriver();
+  std::string SysRoot = computeSysRoot();
+
+  if (DriverArgs.hasArg(clang::driver::options::OPT_nostdinc))
+    return;
+
+  if (!DriverArgs.hasArg(options::OPT_nostdlibinc))
+    addSystemInclude(DriverArgs, CC1Args, SysRoot + "/usr/local/include");
+
+  // Add 'include' in the resource directory, which is similar to
+  // GCC_INCLUDE_DIR (private headers) in GCC.
+  if (!DriverArgs.hasArg(options::OPT_nobuiltininc)) {
+    SmallString<128> ResourceDirInclude(D.ResourceDir);
+    llvm::sys::path::append(ResourceDirInclude, "include");
+    addSystemInclude(DriverArgs, CC1Args, ResourceDirInclude);
+  }
+
+  if (DriverArgs.hasArg(options::OPT_nostdlibinc))
+    return;
+
+  // TOOL_INCLUDE_DIR
+  AddMultilibIncludeArgs(DriverArgs, CC1Args);
+
+  // Check for configure-time C include directories.
+  StringRef CIncludeDirs(C_INCLUDE_DIRS);
+  if (CIncludeDirs != "") {
+    SmallVector<StringRef, 5> dirs;
+    CIncludeDirs.split(dirs, ":");
+    for (StringRef dir : dirs) {
+      StringRef Prefix =
+          llvm::sys::path::is_absolute(dir) ? StringRef(SysRoot) : "";
+      addExternCSystemInclude(DriverArgs, CC1Args, Prefix + dir);
+    }
+    return;
+  }
+
+  // On systems using multiarch, add /usr/include/$triple before
+  // /usr/include.
+  std::string MultiarchIncludeDir = getMultiarchTriple(D, getTriple(), SysRoot);
+  if (!MultiarchIncludeDir.empty())
+    addExternCSystemInclude(
+        DriverArgs, CC1Args,
+        concat(SysRoot, "/usr/include", MultiarchIncludeDir));
+
+  // Add an include of '/include' directly. This isn't provided by default by
+  // system GCCs, but is often used with cross-compiling GCCs, and harmless to
+  // add even when Clang is acting as-if it were a system compiler.
+  addExternCSystemInclude(DriverArgs, CC1Args, concat(SysRoot, "/include"));
+
+  addExternCSystemInclude(DriverArgs, CC1Args, concat(SysRoot, "/usr/include"));
+}
+
+void Managarm::addLibStdCxxIncludePaths(
+    const llvm::opt::ArgList &DriverArgs,
+    llvm::opt::ArgStringList &CC1Args) const {
+  // We need a detected GCC installation on Managarm to provide libstdc++'s
+  // headers.
+  if (!GCCInstallation.isValid())
+    return;
+
+  StringRef TripleStr = GCCInstallation.getTriple().str();
+
+  // Try generic GCC detection.
+  Generic_GCC::addGCCLibStdCxxIncludePaths(DriverArgs, CC1Args, TripleStr);
+}
+
+SanitizerMask Managarm::getSupportedSanitizers() const {
+  const bool IsX86_64 = getTriple().getArch() == llvm::Triple::x86_64;
+  SanitizerMask Res = ToolChain::getSupportedSanitizers();
+  Res |= SanitizerKind::PointerCompare;
+  Res |= SanitizerKind::PointerSubtract;
+  Res |= SanitizerKind::KernelAddress;
+  Res |= SanitizerKind::Vptr;
+  if (IsX86_64)
+    Res |= SanitizerKind::KernelMemory;
+  return Res;
+}
+
+void Managarm::addExtraOpts(llvm::opt::ArgStringList &CmdArgs) const {
+  for (const auto &Opt : ExtraOpts)
+    CmdArgs.push_back(Opt.c_str());
+}
diff --git a/clang/lib/Driver/ToolChains/Managarm.h b/clang/lib/Driver/ToolChains/Managarm.h
new file mode 100644
index 000000000000..2082e2c615f2
--- /dev/null
+++ b/clang/lib/Driver/ToolChains/Managarm.h
@@ -0,0 +1,55 @@
+//===--- Managarm.h - Managarm ToolChain Implementations --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_MANAGARM_H
+#define LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_MANAGARM_H
+
+#include "Gnu.h"
+#include "clang/Driver/ToolChain.h"
+
+namespace clang {
+namespace driver {
+namespace toolchains {
+
+class LLVM_LIBRARY_VISIBILITY Managarm : public Generic_ELF {
+public:
+  Managarm(const Driver &D, const llvm::Triple &Triple,
+           const llvm::opt::ArgList &Args);
+
+  bool HasNativeLLVMSupport() const override;
+
+  std::string getMultiarchTriple(const Driver &D,
+                                 const llvm::Triple &TargetTriple,
+                                 StringRef SysRoot) const override;
+
+  void
+  AddClangSystemIncludeArgs(const llvm::opt::ArgList &DriverArgs,
+                            llvm::opt::ArgStringList &CC1Args) const override;
+  void
+  addLibStdCxxIncludePaths(const llvm::opt::ArgList &DriverArgs,
+                           llvm::opt::ArgStringList &CC1Args) const override;
+  SanitizerMask getSupportedSanitizers() const override;
+  std::string computeSysRoot() const override;
+
+  std::string getDynamicLinker(const llvm::opt::ArgList &Args) const override;
+
+  void addExtraOpts(llvm::opt::ArgStringList &CmdArgs) const override;
+
+  std::vector<std::string> ExtraOpts;
+
+protected:
+  Tool *buildAssembler() const override;
+  Tool *buildLinker() const override;
+};
+
+} // end namespace toolchains
+} // end namespace driver
+} // end namespace clang
+
+#endif // LLVM_CLANG_LIB_DRIVER_TOOLCHAINS_MANAGARM_H
diff --git a/clang/lib/Lex/InitHeaderSearch.cpp b/clang/lib/Lex/InitHeaderSearch.cpp
index 641e3beebc08..3e22b4001bde 100644
--- a/clang/lib/Lex/InitHeaderSearch.cpp
+++ b/clang/lib/Lex/InitHeaderSearch.cpp
@@ -221,6 +221,7 @@ bool InitHeaderSearch::ShouldAddDefaultIncludePaths(
   case llvm::Triple::Hurd:
   case llvm::Triple::Linux:
   case llvm::Triple::LiteOS:
+  case llvm::Triple::Managarm:
   case llvm::Triple::NaCl:
   case llvm::Triple::NetBSD:
   case llvm::Triple::OpenBSD:
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib/aarch64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib/aarch64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib/riscv64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib/riscv64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib/x86_64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib/x86_64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib64/aarch64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib64/aarch64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib64/riscv64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib64/riscv64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/lib64/x86_64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/lib64/x86_64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/aarch64-managarm-mlibc/c++/10/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/aarch64-managarm-mlibc/c++/10/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/10/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/10/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/v1/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/c++/v1/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/riscv64-managarm-mlibc/c++/10/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/riscv64-managarm-mlibc/c++/10/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/x86_64-managarm-mlibc/c++/10/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/include/x86_64-managarm-mlibc/c++/10/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/aarch64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/aarch64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginS.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginS.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginT.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginT.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginS.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginS.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginT.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginT.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginS.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginS.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginT.o b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginT.o
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/riscv64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/riscv64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/x86_64-managarm-mlibc/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib/x86_64-managarm-mlibc/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib64/.keep b/clang/test/Driver/Inputs/basic_managarm_tree/usr/lib64/.keep
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/clang/test/Driver/managarm.cpp b/clang/test/Driver/managarm.cpp
new file mode 100644
index 000000000000..9c3f2d4d722a
--- /dev/null
+++ b/clang/test/Driver/managarm.cpp
@@ -0,0 +1,267 @@
+// UNSUPPORTED: system-windows
+
+// RUN: %clang -### %s --target=x86_64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform 2>&1 | FileCheck --check-prefix=CHECK-X86-64 %s
+// CHECK-X86-64:      "-cc1"
+// CHECK-X86-64-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-X86-64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/x86_64-managarm-mlibc/c++/10"
+// CHECK-X86-64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-X86-64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-X86-64-SAME: "-internal-externc-isystem"
+// CHECK-X86-64-SAME: {{^}} "[[SYSROOT]]/usr/include/x86_64-managarm-mlibc"
+// CHECK-X86-64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-X86-64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-X86-64:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-SAME: "-dynamic-linker" "/lib/x86_64-managarm/ld.so"
+// CHECK-X86-64-SAME: "{{.*}}/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o"
+// CHECK-X86-64-SAME: "-L
+// CHECK-X86-64-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../lib64"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-X86-64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=x86_64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=libc++ --rtlib=compiler-rt --unwindlib=libunwind 2>&1 | FileCheck --check-prefix=CHECK-X86-64-LIBS %s
+// CHECK-X86-64-LIBS:      "-cc1"
+// CHECK-X86-64-LIBS-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/include/c++/v1"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/local/include"
+// CHECK-X86-64-LIBS-SAME: "-internal-externc-isystem"
+// CHECK-X86-64-LIBS-SAME: {{^}} "[[SYSROOT]]/usr/include/x86_64-managarm-mlibc"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-X86-64-LIBS:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-LIBS-SAME: "-dynamic-linker" "/lib/x86_64-managarm/ld.so"
+// CHECK-X86-64-LIBS-SAME: "{{.*}}/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbegin.o"
+// CHECK-X86-64-LIBS-SAME: "-L
+// CHECK-X86-64-LIBS-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../lib64"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-X86-64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=x86_64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform -static 2>&1 | FileCheck --check-prefix=CHECK-X86-64-STATIC %s
+// CHECK-X86-64-STATIC:      "-cc1"
+// CHECK-X86-64-STATIC-SAME: "-static-define"
+// CHECK-X86-64-STATIC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/x86_64-managarm-mlibc/c++/10"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-X86-64-STATIC-SAME: "-internal-externc-isystem"
+// CHECK-X86-64-STATIC-SAME: {{^}} "[[SYSROOT]]/usr/include/x86_64-managarm-mlibc"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-X86-64-STATIC:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-STATIC-SAME: "-static"
+// CHECK-X86-64-STATIC-SAME: "{{.*}}/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginT.o"
+// CHECK-X86-64-STATIC-SAME: "-L
+// CHECK-X86-64-STATIC-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../lib64"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-X86-64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=x86_64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   -shared 2>&1 | FileCheck --check-prefix=CHECK-X86-64-SHARED %s
+// CHECK-X86-64-SHARED:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-X86-64-SHARED-SAME: "{{.*}}/usr/lib/gcc/x86_64-managarm-mlibc/10/crtbeginS.o"
+// CHECK-X86-64-SHARED-SAME: "-L
+// CHECK-X86-64-SHARED-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/x86_64-managarm-mlibc/10/../../../../lib64"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/x86_64-managarm-mlibc"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-X86-64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=aarch64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform 2>&1 | FileCheck --check-prefix=CHECK-AARCH64 %s
+// CHECK-AARCH64:      "-cc1"
+// CHECK-AARCH64-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/aarch64-managarm-mlibc/c++/10"
+// CHECK-AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-AARCH64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-AARCH64-SAME: "-internal-externc-isystem"
+// CHECK-AARCH64-SAME: {{^}} "[[SYSROOT]]/usr/include/aarch64-managarm-mlibc"
+// CHECK-AARCH64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-AARCH64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-AARCH64:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-SAME: "-m" "aarch64managarm"
+// CHECK-AARCH64-SAME: {{^}} "-dynamic-linker" "/lib/aarch64-managarm/ld.so"
+// CHECK-AARCH64-SAME: "{{.*}}/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o"
+// CHECK-AARCH64-SAME: {{^}} "-L
+// CHECK-AARCH64-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../lib64"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-AARCH64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=aarch64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=libc++ --rtlib=compiler-rt --unwindlib=libunwind 2>&1 | FileCheck --check-prefix=CHECK-AARCH64-LIBS %s
+// CHECK-AARCH64-LIBS:      "-cc1"
+// CHECK-AARCH64-LIBS-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/include/c++/v1"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/local/include"
+// CHECK-AARCH64-LIBS-SAME: "-internal-externc-isystem"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "[[SYSROOT]]/usr/include/aarch64-managarm-mlibc"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-AARCH64-LIBS:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-LIBS-SAME: "-m" "aarch64managarm"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-dynamic-linker" "/lib/aarch64-managarm/ld.so"
+// CHECK-AARCH64-LIBS-SAME: "{{.*}}/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbegin.o"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L
+// CHECK-AARCH64-LIBS-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../lib64"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-AARCH64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=aarch64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform -static 2>&1 | FileCheck --check-prefix=CHECK-AARCH64-STATIC %s
+// CHECK-AARCH64-STATIC:      "-cc1"
+// CHECK-AARCH64-STATIC-SAME: "-static-define"
+// CHECK-AARCH64-STATIC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/aarch64-managarm-mlibc/c++/10"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-AARCH64-STATIC-SAME: "-internal-externc-isystem"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "[[SYSROOT]]/usr/include/aarch64-managarm-mlibc"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-AARCH64-STATIC:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-STATIC-SAME: "-m" "aarch64managarm"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-static"
+// CHECK-AARCH64-STATIC-SAME: "{{.*}}/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginT.o"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L
+// CHECK-AARCH64-STATIC-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../lib64"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-AARCH64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=aarch64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   -shared 2>&1 | FileCheck --check-prefix=CHECK-AARCH64-SHARED %s
+// CHECK-AARCH64-SHARED:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-AARCH64-SHARED-SAME: "-m" "aarch64managarm"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "{{.*}}/usr/lib/gcc/aarch64-managarm-mlibc/10/crtbeginS.o"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L
+// CHECK-AARCH64-SHARED-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/aarch64-managarm-mlibc/10/../../../../lib64"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/aarch64-managarm-mlibc"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-AARCH64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=riscv64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform 2>&1 | FileCheck --check-prefix=CHECK-RISCV64 %s
+// CHECK-RISCV64:      "-cc1"
+// CHECK-RISCV64-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-RISCV64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/riscv64-managarm-mlibc/c++/10"
+// CHECK-RISCV64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-RISCV64-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-RISCV64-SAME: "-internal-externc-isystem"
+// CHECK-RISCV64-SAME: {{^}} "[[SYSROOT]]/usr/include/riscv64-managarm-mlibc"
+// CHECK-RISCV64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-RISCV64-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-RISCV64:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-SAME: "{{.*}}/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o"
+// CHECK-RISCV64-SAME: "-L
+// CHECK-RISCV64-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../lib64"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-RISCV64-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=riscv64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=libc++ --rtlib=compiler-rt --unwindlib=libunwind 2>&1 | FileCheck --check-prefix=CHECK-RISCV64-LIBS %s
+// CHECK-RISCV64-LIBS:      "-cc1"
+// CHECK-RISCV64-LIBS-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/include/c++/v1"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-internal-isystem" "[[SYSROOT:[^"]+]]/usr/local/include"
+// CHECK-RISCV64-LIBS-SAME: "-internal-externc-isystem"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "[[SYSROOT]]/usr/include/riscv64-managarm-mlibc"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-RISCV64-LIBS:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-LIBS-SAME: "{{.*}}/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbegin.o"
+// CHECK-RISCV64-LIBS-SAME: "-L
+// CHECK-RISCV64-LIBS-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../lib64"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-RISCV64-LIBS-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=riscv64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   --stdlib=platform -static 2>&1 | FileCheck --check-prefix=CHECK-RISCV64-STATIC %s
+// CHECK-RISCV64-STATIC:      "-cc1"
+// CHECK-RISCV64-STATIC-SAME: "-static-define"
+// CHECK-RISCV64-STATIC-SAME: "-isysroot" "[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/c++/10"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/riscv64-managarm-mlibc/c++/10"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../include/c++/10/backward"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-isystem" "[[SYSROOT]]/usr/local/include"
+// CHECK-RISCV64-STATIC-SAME: "-internal-externc-isystem"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "[[SYSROOT]]/usr/include/riscv64-managarm-mlibc"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/include"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-internal-externc-isystem" "[[SYSROOT]]/usr/include"
+// CHECK-RISCV64-STATIC:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-STATIC-SAME: "-static"
+// CHECK-RISCV64-STATIC-SAME: "{{.*}}/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginT.o"
+// CHECK-RISCV64-STATIC-SAME: "-L
+// CHECK-RISCV64-STATIC-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../lib64"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-RISCV64-STATIC-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
+
+// RUN: %clang -### %s --target=riscv64-unknown-managarm-mlibc --sysroot=%S/Inputs/basic_managarm_tree \
+// RUN:   -shared 2>&1 | FileCheck --check-prefix=CHECK-RISCV64-SHARED %s
+// CHECK-RISCV64-SHARED:      "{{.*}}ld" "--sysroot=[[SYSROOT:[^"]+]]"
+// CHECK-RISCV64-SHARED-SAME: "{{.*}}/usr/lib/gcc/riscv64-managarm-mlibc/10/crtbeginS.o"
+// CHECK-RISCV64-SHARED-SAME: "-L
+// CHECK-RISCV64-SHARED-SAME: {{^}}[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/gcc/riscv64-managarm-mlibc/10/../../../../lib64"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib/../lib64"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib/riscv64-managarm-mlibc"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib64"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/lib"
+// CHECK-RISCV64-SHARED-SAME: {{^}} "-L[[SYSROOT]]/usr/lib"
diff --git a/clang/test/Preprocessor/init.c b/clang/test/Preprocessor/init.c
index 031a6c1a755b..bed39dc3e34d 100644
--- a/clang/test/Preprocessor/init.c
+++ b/clang/test/Preprocessor/init.c
@@ -1622,6 +1622,11 @@
 // RUN: %clang_cc1 -x c -std=c99 -E -dM -ffreestanding -triple=amd64-unknown-openbsd < /dev/null | FileCheck -match-full-lines -check-prefix OPENBSD-STDC-N %s
 // OPENBSD-STDC-N-NOT:#define __STDC_NO_THREADS__ 1
 //
+// RUN: %clang_cc1 -triple=aarch64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
+// RUN: %clang_cc1 -triple=riscv64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
+// RUN: %clang_cc1 -triple=x86_64-unknown-managarm-mlibc -E -dM < /dev/null | FileCheck -match-full-lines -check-prefix MANAGARM %s
+// MANAGARM: #define __managarm__ 1
+
 // RUN: %clang_cc1 -E -dM -ffreestanding -triple=xcore-none-none < /dev/null | FileCheck -match-full-lines -check-prefix XCORE %s
 // XCORE:#define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__
 // XCORE:#define __LITTLE_ENDIAN__ 1
diff --git a/clang/test/Preprocessor/predefined-macros-no-warnings.c b/clang/test/Preprocessor/predefined-macros-no-warnings.c
index 4e3e29ccfa8a..fe27ed8814ee 100644
--- a/clang/test/Preprocessor/predefined-macros-no-warnings.c
+++ b/clang/test/Preprocessor/predefined-macros-no-warnings.c
@@ -14,6 +14,7 @@
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-fuchsia
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-linux
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-linux-openhos
+// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-managarm
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-netbsd
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-openbsd
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple aarch64-win32-gnu
@@ -108,6 +109,7 @@
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple riscv64-fuchsia
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple riscv64-linux
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple riscv64-linux-openhos
+// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple riscv64-managarm
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple sparc
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple sparc-linux
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple sparc-solaris
@@ -167,6 +169,7 @@
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple x86_64-nacl
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple x86_64-ps4
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple x86_64-ps5
+// RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple x86_64-managarm
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spir
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spir64
 // RUN: %clang_cc1 %s -Eonly -Wsystem-headers -Werror -triple spirv32
-- 
cgit v1.2.3


From 06e08f38e1ec57bf3cb5e08569b52eb6a3c3e166 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 15:43:39 +0900
Subject: AArch64: Use reportFatalUsageError for unsupported calling conv
 (#144840)

This probably should emit a DiagnosticInfoUnsupported and use the
default calling convention instead, but then we would need to pass
in the context.

Also move where CCAssignFnForCall is called. It was unnecessarily
called for each argument, so the error wouldn't trigger for functions
with 0 arguments.

This only ensures the error occurs for functions defined with the
calling convention. The error is still missed for outgoing calls
with no arguments. The lowering logic here is convoluted, calling
CCAssignFnForCall for each argument and it does not mirror
LowerFormalArguments so I'm not sure what's going on here.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp  | 14 +++++++++-----
 llvm/test/CodeGen/AArch64/unsupported-cc-call.ll |  8 ++++++++
 llvm/test/CodeGen/AArch64/unsupported-cc-func.ll |  6 ++++++
 llvm/test/MC/AArch64/coff-function-type-info.ll  |  6 +++---
 4 files changed, 26 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/unsupported-cc-call.ll
 create mode 100644 llvm/test/CodeGen/AArch64/unsupported-cc-func.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 581f15277602..69f8c6c21a55 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7632,7 +7632,7 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
                                                      bool IsVarArg) const {
   switch (CC) {
   default:
-    report_fatal_error("Unsupported calling convention.");
+    reportFatalUsageError("unsupported calling convention");
   case CallingConv::GHC:
     return CC_AArch64_GHC;
   case CallingConv::PreserveNone:
@@ -7741,6 +7741,12 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
   unsigned NumArgs = Ins.size();
   Function::const_arg_iterator CurOrigArg = F.arg_begin();
   unsigned CurArgIdx = 0;
+  bool UseVarArgCC = false;
+  if (IsWin64)
+    UseVarArgCC = isVarArg;
+
+  CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC);
+
   for (unsigned i = 0; i != NumArgs; ++i) {
     MVT ValVT = Ins[i].VT;
     if (Ins[i].isOrigArg()) {
@@ -7757,10 +7763,6 @@ SDValue AArch64TargetLowering::LowerFormalArguments(
       else if (ActualMVT == MVT::i16)
         ValVT = MVT::i16;
     }
-    bool UseVarArgCC = false;
-    if (IsWin64)
-      UseVarArgCC = isVarArg;
-    CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, UseVarArgCC);
     bool Res =
         AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
     assert(!Res && "Call operand has unhandled type");
@@ -8429,6 +8431,8 @@ static void analyzeCallOperands(const AArch64TargetLowering &TLI,
         ArgVT = MVT::i16;
     }
 
+    // FIXME: CCAssignFnForCall should be called once, for the call and not per
+    // argument. This logic should exactly mirror LowerFormalArguments.
     CCAssignFn *AssignFn = TLI.CCAssignFnForCall(CalleeCC, UseVarArgCC);
     bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
     assert(!Res && "Call operand has unhandled type");
diff --git a/llvm/test/CodeGen/AArch64/unsupported-cc-call.ll b/llvm/test/CodeGen/AArch64/unsupported-cc-call.ll
new file mode 100644
index 000000000000..3e03ac81baf5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/unsupported-cc-call.ll
@@ -0,0 +1,8 @@
+; FIXME: This should error:
+; RUN: llc -mtriple=aarch64-- -filetype=null %s
+declare amdgpu_gfx void @amdgpu_gfx_func()
+
+define void @call_amdgpu_gfx_func() {
+  call amdgpu_gfx void @amdgpu_gfx_func()
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/unsupported-cc-func.ll b/llvm/test/CodeGen/AArch64/unsupported-cc-func.ll
new file mode 100644
index 000000000000..2d0fb12af0d4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/unsupported-cc-func.ll
@@ -0,0 +1,6 @@
+; RUN: not llc -mtriple=aarch64-- -filetype=null %s 2>&1 | FileCheck %s
+
+; CHECK: LLVM ERROR: unsupported calling convention
+define amdgpu_gfx void @amdgpu_gfx_func_definition() {
+  ret void
+}
diff --git a/llvm/test/MC/AArch64/coff-function-type-info.ll b/llvm/test/MC/AArch64/coff-function-type-info.ll
index 21446c50ab27..63cac59adef1 100644
--- a/llvm/test/MC/AArch64/coff-function-type-info.ll
+++ b/llvm/test/MC/AArch64/coff-function-type-info.ll
@@ -1,10 +1,10 @@
-; RUN: llc -mtriple arm64-windows -filetype asm -o - %s \
+    ; RUN: llc -mtriple arm64-windows -filetype asm -o - %s \
 ; RUN:    | FileCheck %s -check-prefix CHECK-ASM
 
 ; RUN: llc -mtriple arm64-windows -filetype obj -o - %s \
 ; RUN:    | llvm-readobj --symbols - | FileCheck %s -check-prefix CHECK-OBJECT
 
-define arm_aapcs_vfpcc void @external() {
+define aarch64_vector_pcs void @external() {
 entry:
   ret void
 }
@@ -15,7 +15,7 @@ entry:
 ; CHECK-ASM: .endef
 ; CHECK-ASM: .globl external
 
-define internal arm_aapcs_vfpcc void @internal() {
+define internal aarch64_vector_pcs void @internal() {
 entry:
   ret void
 }
-- 
cgit v1.2.3


From 090f409538d2b426f11ce5aa22af8c243099aecf Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 20 Jun 2025 06:47:44 +0000
Subject: [gn build] Port 0f302f38b001

---
 llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
index e929e0088384..98c78dc15b32 100644
--- a/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Driver/BUILD.gn
@@ -84,6 +84,7 @@ static_library("Driver") {
     "ToolChains/Linux.cpp",
     "ToolChains/MSP430.cpp",
     "ToolChains/MSVC.cpp",
+    "ToolChains/Managarm.cpp",
     "ToolChains/MinGW.cpp",
     "ToolChains/MipsLinux.cpp",
     "ToolChains/NaCl.cpp",
-- 
cgit v1.2.3


From d196124dd22391f6c967ed569b34632840536c45 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 20 Jun 2025 09:16:39 +0200
Subject: [PredicateInfo] Remove unnecessary EdgeUsesOnly set (NFC) (#144912)

As far as I can tell, this set is pointless: It just represents whether
the target block has multiple predecessors, and the way it is
constructed and queried, we're not even reducing the number of
getSinglePredecessor() calls or something like that.
---
 llvm/lib/Transforms/Utils/PredicateInfo.cpp | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 4c87babbfb6f..4a0faab00cca 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -253,10 +253,6 @@ class PredicateInfoBuilder {
   // whether it returned a valid result.
   DenseMap<Value *, unsigned int> ValueInfoNums;
 
-  // The set of edges along which we can only handle phi uses, due to critical
-  // edges.
-  DenseSet<std::pair<BasicBlock *, BasicBlock *>> EdgeUsesOnly;
-
   ValueInfo &getOrCreateValueInfo(Value *);
   const ValueInfo &getValueInfo(Value *) const;
 
@@ -459,8 +455,6 @@ void PredicateInfoBuilder::processBranch(
           PredicateBase *PB =
               new PredicateBranch(V, BranchBB, Succ, Cond, TakenEdge);
           addInfoFor(OpsToRename, V, PB);
-          if (!Succ->getSinglePredecessor())
-            EdgeUsesOnly.insert({BranchBB, Succ});
         }
       }
     }
@@ -487,8 +481,6 @@ void PredicateInfoBuilder::processSwitch(
       PredicateSwitch *PS = new PredicateSwitch(
           Op, SI->getParent(), TargetBlock, C.getCaseValue(), SI);
       addInfoFor(OpsToRename, Op, PS);
-      if (!TargetBlock->getSinglePredecessor())
-        EdgeUsesOnly.insert({BranchBB, TargetBlock});
     }
   }
 }
@@ -637,7 +629,7 @@ void PredicateInfoBuilder::renameUses(SmallVectorImpl<Value *> &OpsToRename) {
         // block, and handle it specially. We know that it goes last, and only
         // dominate phi uses.
         auto BlockEdge = getBlockEdge(PossibleCopy);
-        if (EdgeUsesOnly.count(BlockEdge)) {
+        if (!BlockEdge.second->getSinglePredecessor()) {
           VD.LocalNum = LN_Last;
           auto *DomNode = DT.getNode(BlockEdge.first);
           if (DomNode) {
-- 
cgit v1.2.3


From cbb5e244f7564091f9169f525fd8456e68bc028a Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 20 Jun 2025 09:35:11 +0200
Subject: [PredicateInfo] Remove redundant EdgeOnly member (NFC)

EdgeOnly indicates a phi def, which can already be identified by
LN_Last with non-null PInfo. Most of the code already reasons in
terms of LN_Last instead of EdgeOnly.
---
 llvm/lib/Transforms/Utils/PredicateInfo.cpp | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 4a0faab00cca..778287bb41b6 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -89,9 +89,7 @@ struct ValueDFS {
   // Only one of Def or Use will be set.
   Value *Def = nullptr;
   Use *U = nullptr;
-  // Neither PInfo nor EdgeOnly participate in the ordering
   PredicateBase *PInfo = nullptr;
-  bool EdgeOnly = false;
 };
 
 // Perform a strict weak ordering on instructions and arguments.
@@ -289,10 +287,11 @@ bool PredicateInfoBuilder::stackIsInScope(const ValueDFSStack &Stack,
     return false;
   // If it's a phi only use, make sure it's for this phi node edge, and that the
   // use is in a phi node.  If it's anything else, and the top of the stack is
-  // EdgeOnly, we need to pop the stack.  We deliberately sort phi uses next to
-  // the defs they must go with so that we can know it's time to pop the stack
-  // when we hit the end of the phi uses for a given def.
-  if (Stack.back().EdgeOnly) {
+  // a LN_Last def, we need to pop the stack.  We deliberately sort phi uses
+  // next to the defs they must go with so that we can know it's time to pop
+  // the stack when we hit the end of the phi uses for a given def.
+  const ValueDFS &Top = Stack.back();
+  if (Top.LocalNum == LN_Last && Top.PInfo) {
     if (!VDUse.U)
       return false;
     auto *PHI = dyn_cast<PHINode>(VDUse.U->getUser());
@@ -300,15 +299,14 @@ bool PredicateInfoBuilder::stackIsInScope(const ValueDFSStack &Stack,
       return false;
     // Check edge
     BasicBlock *EdgePred = PHI->getIncomingBlock(*VDUse.U);
-    if (EdgePred != getBranchBlock(Stack.back().PInfo))
+    if (EdgePred != getBranchBlock(Top.PInfo))
       return false;
 
     // Use dominates, which knows how to handle edge dominance.
-    return DT.dominates(getBlockEdge(Stack.back().PInfo), *VDUse.U);
+    return DT.dominates(getBlockEdge(Top.PInfo), *VDUse.U);
   }
 
-  return (VDUse.DFSIn >= Stack.back().DFSIn &&
-          VDUse.DFSOut <= Stack.back().DFSOut);
+  return VDUse.DFSIn >= Top.DFSIn && VDUse.DFSOut <= Top.DFSOut;
 }
 
 void PredicateInfoBuilder::popStackUntilDFSScope(ValueDFSStack &Stack,
@@ -636,7 +634,6 @@ void PredicateInfoBuilder::renameUses(SmallVectorImpl<Value *> &OpsToRename) {
             VD.DFSIn = DomNode->getDFSNumIn();
             VD.DFSOut = DomNode->getDFSNumOut();
             VD.PInfo = PossibleCopy;
-            VD.EdgeOnly = true;
             OrderedUses.push_back(VD);
           }
         } else {
-- 
cgit v1.2.3


From 1cae21da47b1f53c3946534b12507a035fb283d2 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 16:43:39 +0900
Subject: AMDGPU: Remove legacy PM version of AMDGPUPromoteAllocaToVector
 (#144986)

This is only run in the middle end with the new pass manager now,
so garbage collect the old PM version.
---
 llvm/lib/Target/AMDGPU/AMDGPU.h                |  4 ---
 llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 40 --------------------------
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 3 files changed, 45 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 5a917734e9c7..0ad18c38213f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -242,10 +242,6 @@ FunctionPass *createAMDGPUPromoteAlloca();
 void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
 extern char &AMDGPUPromoteAllocaID;
 
-FunctionPass *createAMDGPUPromoteAllocaToVector();
-void initializeAMDGPUPromoteAllocaToVectorPass(PassRegistry&);
-extern char &AMDGPUPromoteAllocaToVectorID;
-
 struct AMDGPUPromoteAllocaPass : PassInfoMixin<AMDGPUPromoteAllocaPass> {
   AMDGPUPromoteAllocaPass(TargetMachine &TM) : TM(TM) {}
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
index 700dc87d2f82..e0f3c72890b0 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp
@@ -167,34 +167,6 @@ public:
   }
 };
 
-class AMDGPUPromoteAllocaToVector : public FunctionPass {
-public:
-  static char ID;
-
-  AMDGPUPromoteAllocaToVector() : FunctionPass(ID) {}
-
-  bool runOnFunction(Function &F) override {
-    if (skipFunction(F))
-      return false;
-    if (auto *TPC = getAnalysisIfAvailable<TargetPassConfig>())
-      return AMDGPUPromoteAllocaImpl(
-                 TPC->getTM<TargetMachine>(),
-                 getAnalysis<LoopInfoWrapperPass>().getLoopInfo())
-          .run(F, /*PromoteToLDS*/ false);
-    return false;
-  }
-
-  StringRef getPassName() const override {
-    return "AMDGPU Promote Alloca to vector";
-  }
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesCFG();
-    AU.addRequired<LoopInfoWrapperPass>();
-    FunctionPass::getAnalysisUsage(AU);
-  }
-};
-
 static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
                             const Function &F) {
   if (!TM.getTargetTriple().isAMDGCN())
@@ -216,7 +188,6 @@ static unsigned getMaxVGPRs(unsigned LDSBytes, const TargetMachine &TM,
 } // end anonymous namespace
 
 char AMDGPUPromoteAlloca::ID = 0;
-char AMDGPUPromoteAllocaToVector::ID = 0;
 
 INITIALIZE_PASS_BEGIN(AMDGPUPromoteAlloca, DEBUG_TYPE,
                       "AMDGPU promote alloca to vector or LDS", false, false)
@@ -227,14 +198,7 @@ INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_END(AMDGPUPromoteAlloca, DEBUG_TYPE,
                     "AMDGPU promote alloca to vector or LDS", false, false)
 
-INITIALIZE_PASS_BEGIN(AMDGPUPromoteAllocaToVector, DEBUG_TYPE "-to-vector",
-                      "AMDGPU promote alloca to vector", false, false)
-INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
-INITIALIZE_PASS_END(AMDGPUPromoteAllocaToVector, DEBUG_TYPE "-to-vector",
-                    "AMDGPU promote alloca to vector", false, false)
-
 char &llvm::AMDGPUPromoteAllocaID = AMDGPUPromoteAlloca::ID;
-char &llvm::AMDGPUPromoteAllocaToVectorID = AMDGPUPromoteAllocaToVector::ID;
 
 PreservedAnalyses AMDGPUPromoteAllocaPass::run(Function &F,
                                                FunctionAnalysisManager &AM) {
@@ -264,10 +228,6 @@ FunctionPass *llvm::createAMDGPUPromoteAlloca() {
   return new AMDGPUPromoteAlloca();
 }
 
-FunctionPass *llvm::createAMDGPUPromoteAllocaToVector() {
-  return new AMDGPUPromoteAllocaToVector();
-}
-
 static void collectAllocaUses(AllocaInst &Alloca,
                               SmallVectorImpl<Use *> &Uses) {
   SmallVector<Instruction *, 4> WorkList({&Alloca});
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index f390d39043ed..074beccef33c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -529,7 +529,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUPreLegalizerCombinerPass(*PR);
   initializeAMDGPURegBankCombinerPass(*PR);
   initializeAMDGPUPromoteAllocaPass(*PR);
-  initializeAMDGPUPromoteAllocaToVectorPass(*PR);
   initializeAMDGPUCodeGenPreparePass(*PR);
   initializeAMDGPULateCodeGenPrepareLegacyPass(*PR);
   initializeAMDGPURemoveIncompatibleFunctionsLegacyPass(*PR);
-- 
cgit v1.2.3


From c361bffa50f1ed790c393ffbab39c2e07dfcb242 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 16:44:06 +0900
Subject: AMDGPU: Remove legacy pass manager version of AMDGPUUnifyMetadata
 (#144985)

This is only run in the new pass manager now.
---
 llvm/lib/Target/AMDGPU/AMDGPU.h                |  4 ----
 llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |  1 -
 llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp | 27 +-------------------------
 3 files changed, 1 insertion(+), 31 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 0ad18c38213f..68a3caf59544 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -439,10 +439,6 @@ struct AMDGPUPrintfRuntimeBindingPass
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
 
-ModulePass* createAMDGPUUnifyMetadataPass();
-void initializeAMDGPUUnifyMetadataPass(PassRegistry&);
-extern char &AMDGPUUnifyMetadataID;
-
 struct AMDGPUUnifyMetadataPass : PassInfoMixin<AMDGPUUnifyMetadataPass> {
   PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
 };
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 074beccef33c..1ea7dd01d15c 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -537,7 +537,6 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUReserveWWMRegsLegacyPass(*PR);
   initializeAMDGPURewriteOutArgumentsPass(*PR);
   initializeAMDGPURewriteUndefForPHILegacyPass(*PR);
-  initializeAMDGPUUnifyMetadataPass(*PR);
   initializeSIAnnotateControlFlowLegacyPass(*PR);
   initializeAMDGPUInsertDelayAluLegacyPass(*PR);
   initializeSIInsertHardClausesLegacyPass(*PR);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
index cd7866b86d55..e400491c3860 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp
@@ -32,17 +32,6 @@ namespace {
 
   } // end namespace kOCLMD
 
-  /// Unify multiple OpenCL metadata due to linking.
-  class AMDGPUUnifyMetadata : public ModulePass {
-  public:
-    static char ID;
-
-    explicit AMDGPUUnifyMetadata() : ModulePass(ID) {}
-
-  private:
-    bool runOnModule(Module &M) override;
-  };
-
     /// Unify version metadata.
     /// \return true if changes are made.
     /// Assume the named metadata has operands each of which is a pair of
@@ -104,6 +93,7 @@ namespace {
     return true;
   }
 
+  /// Unify multiple OpenCL metadata due to linking.
   bool unifyMetadataImpl(Module &M) {
     const char *Vers[] = {kOCLMD::SpirVer, kOCLMD::OCLVer};
     const char *Exts[] = {kOCLMD::UsedExt, kOCLMD::UsedOptCoreFeat,
@@ -122,21 +112,6 @@ namespace {
 
   } // end anonymous namespace
 
-  char AMDGPUUnifyMetadata::ID = 0;
-
-  char &llvm::AMDGPUUnifyMetadataID = AMDGPUUnifyMetadata::ID;
-
-  INITIALIZE_PASS(AMDGPUUnifyMetadata, "amdgpu-unify-metadata",
-                  "Unify multiple OpenCL metadata due to linking", false, false)
-
-  ModulePass *llvm::createAMDGPUUnifyMetadataPass() {
-    return new AMDGPUUnifyMetadata();
-  }
-
-  bool AMDGPUUnifyMetadata::runOnModule(Module &M) {
-    return unifyMetadataImpl(M);
-  }
-
   PreservedAnalyses AMDGPUUnifyMetadataPass::run(Module &M,
                                                  ModuleAnalysisManager &AM) {
     return unifyMetadataImpl(M) ? PreservedAnalyses::none()
-- 
cgit v1.2.3


From 8973be462c49a7b0a24c61f41e07a721706b1ad8 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 20 Jun 2025 09:46:41 +0200
Subject: [PredicateInfo] Avoid duplicate stack in scope check (NFC)

popStackUntilDFSScope() is going to check this itself, there is
no need to do it in advance as well.
---
 llvm/lib/Transforms/Utils/PredicateInfo.cpp | 21 ++++++++-------------
 1 file changed, 8 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 778287bb41b6..9b239d9161e7 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -283,8 +283,7 @@ public:
 
 bool PredicateInfoBuilder::stackIsInScope(const ValueDFSStack &Stack,
                                           const ValueDFS &VDUse) const {
-  if (Stack.empty())
-    return false;
+  assert(!Stack.empty() && "Should not be called with empty stack");
   // If it's a phi only use, make sure it's for this phi node edge, and that the
   // use is in a phi node.  If it's anything else, and the top of the stack is
   // a LN_Last def, we need to pop the stack.  We deliberately sort phi uses
@@ -677,22 +676,18 @@ void PredicateInfoBuilder::renameUses(SmallVectorImpl<Value *> &OpsToRename) {
       LLVM_DEBUG(dbgs() << "Current DFS numbers are (" << VD.DFSIn << ","
                         << VD.DFSOut << ")\n");
 
-      bool ShouldPush = (VD.Def || PossibleCopy);
-      bool OutOfScope = !stackIsInScope(RenameStack, VD);
-      if (OutOfScope || ShouldPush) {
-        // Sync to our current scope.
-        popStackUntilDFSScope(RenameStack, VD);
-        if (ShouldPush) {
-          RenameStack.push_back(VD);
-        }
+      // Sync to our current scope.
+      popStackUntilDFSScope(RenameStack, VD);
+
+      if (VD.Def || PossibleCopy) {
+        RenameStack.push_back(VD);
+        continue;
       }
+
       // If we get to this point, and the stack is empty we must have a use
       // with no renaming needed, just skip it.
       if (RenameStack.empty())
         continue;
-      // Skip values, only want to rename the uses
-      if (VD.Def || PossibleCopy)
-        continue;
       if (!DebugCounter::shouldExecute(RenameCounter)) {
         LLVM_DEBUG(dbgs() << "Skipping execution due to debug counter\n");
         continue;
-- 
cgit v1.2.3


From bc14e5e5e9c717a7699cf10b13a7661a9e033594 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 01:05:43 -0700
Subject: [AMDGPU] Add trivial gfx1250 runlines to MC tests. NFC. (#144988)

---
 llvm/test/MC/AMDGPU/gfx12_asm_smem.s                                 | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_sopk.s                                 | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s                           | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s                            | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s                           | 2 ++
 llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s                            | 2 ++
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s                 | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s                  | 1 +
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s                 | 2 ++
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s                  | 2 ++
 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt                 | 1 +
 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt            | 2 ++
 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt | 2 ++
 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt  | 2 ++
 14 files changed, 21 insertions(+)

diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s
index 2ef027459fa6..f4086e81d5c6 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_smem.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_smem.s
@@ -1,4 +1,5 @@
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1250 %s | FileCheck --check-prefix=GFX12 %s
 
 //===----------------------------------------------------------------------===//
 // ENC_SMEM.
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s
index 5ce6847b9dca..4e3e725a0055 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopk.s
@@ -1,4 +1,5 @@
 // RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1250 %s | FileCheck --check-prefix=GFX12 %s
 
 s_movk_i32 s0, 0x1234
 // GFX12: encoding: [0x34,0x12,0x00,0xb0]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s
index 9ccdd2f604cb..06ccf098bd60 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp16.s
@@ -1,6 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX12: v_bfrev_b32_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x70,0x0a,0x7e,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s
index 61ee43dc9d16..d28d00da1910 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop1_dpp8.s
@@ -1,6 +1,7 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12 %s
 
 v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
index d4378ae0a210..3b10481d5e72 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp16.s
@@ -3,6 +3,8 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-wavefrontsize32,+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0]
 // W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0x0a,0x40,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
index a81cd58acf03..235a6ba3dbb9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop2_dpp8.s
@@ -3,6 +3,8 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-wavefrontsize32,+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
 // W32: v_add_co_ci_u32_dpp v5, vcc_lo, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x04,0x0a,0x40,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s
index 6d07f299d1d6..292419949aef 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp16.s
@@ -1,5 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
 
 v_bfrev_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0]
 // GFX12: v_bfrev_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s
index d2fc865cc8b4..c61fa9aff244 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop1_dpp8.s
@@ -1,5 +1,6 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
 // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
 
 v_bfrev_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_bfrev_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
index ee28cc85bc5c..8fea02c9029e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp16.s
@@ -3,6 +3,8 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-wavefrontsize32,+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x06,0x20,0xd5,0xfa,0x04,0x0e,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
index 2bdad0875e00..a8e7676a3cb9 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_from_vop2_dpp8.s
@@ -3,6 +3,8 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefix=W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-wavefrontsize32,+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0]
 // W32: v_add_co_ci_u32_e64_dpp v5, s6, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x06,0x20,0xd5,0xe9,0x04,0x0e,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt
index 02641e6eb97f..328fdbca10f3 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_smem.txt
@@ -1,4 +1,5 @@
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
 
 # GFX12: s_load_i8 s5, s[2:3], s0 offset:0x0            ; encoding: [0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00]
 0x41,0x01,0x01,0xf4,0x00,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt
index 7f9b268440cf..7a7be57e17e7 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop1_dpp8.txt
@@ -3,6 +3,8 @@
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 
 0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05
 # GFX12: v_bfrev_b32_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0xe9,0x70,0x0a,0x7e,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt
index a020b0ae46a3..32eb7711089e 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp16.txt
@@ -1,6 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 
 0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff
 # GFX12: v_bfrev_b32_e64_dpp v5, v1 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0xb8,0xd5,0xfa,0x00,0x00,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt
index ad3c673b4e39..13a58aa6b415 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_from_vop1_dpp8.txt
@@ -1,6 +1,8 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
 # RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX12-FAKE16 %s
 
 0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05
 # GFX12: v_bfrev_b32_e64_dpp v5, v1 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0xb8,0xd5,0xe9,0x00,0x00,0x00,0x01,0x77,0x39,0x05]
-- 
cgit v1.2.3


From b6b8fa3b15d334c51fcf8763ccda0102a01aeb9c Mon Sep 17 00:00:00 2001
From: int-zjt <zhangjiatong.0@bytedance.com>
Date: Fri, 20 Jun 2025 16:24:19 +0800
Subject: [llvm-cov][gcov] Support multi-files coverage in one basic block
 (#144504)

In the current gcov implementation, all lines within a basic block are
attributed to the source file of the block's containing function. This
is inaccurate when a block contains lines from other files (e.g., via
#include "foo.inc").

Commit
[406e81b](https://github.com/llvm/llvm-project/commit/406e81b79d26dae6838cc69d10a3e22635da09ef)
attempted to address this by filtering lines based on debug info types,
but this approach has two limitations:

* **Over-filtering**: Some valid lines belonging to the function are
incorrectly excluded.
* **Under-counting**: Lines not belonging to the function are filtered
out and omitted from coverage statistics.

**GCC Reference Behavior**
GCC's gcov implementation handles this case correctly.This change aligns
the LLVM behavior with GCC.

**Proposed Solution**
1. **GCNO Generation**:

* **Current**: Each block stores a single GCOVLines record (filename +
lines).

* **New**: Dynamically create new GCOVLines records whenever consecutive
lines in a block originate from different source files. Group subsequent
lines from the same file under one record.

2. **GCNO Parsing**:

* **Current**: Lines are directly attributed to the function's source
file.

* **New**: Introduce a GCOVLocation type to track filename/line mappings
within blocks. Statistics will reflect the actual source file for each
line.
---
 .../test/profile/Posix/gcov-file-change-line.cpp   | 15 ++++++++
 .../test/profile/Posix/gcov-file-change.cpp        | 12 +++---
 llvm/include/llvm/ProfileData/GCOV.h               | 20 ++++++++--
 llvm/lib/ProfileData/GCOV.cpp                      | 45 +++++++++++++---------
 .../Transforms/Instrumentation/GCOVProfiling.cpp   | 37 ++++++++----------
 5 files changed, 80 insertions(+), 49 deletions(-)
 create mode 100644 compiler-rt/test/profile/Posix/gcov-file-change-line.cpp

diff --git a/compiler-rt/test/profile/Posix/gcov-file-change-line.cpp b/compiler-rt/test/profile/Posix/gcov-file-change-line.cpp
new file mode 100644
index 000000000000..a750befb47e5
--- /dev/null
+++ b/compiler-rt/test/profile/Posix/gcov-file-change-line.cpp
@@ -0,0 +1,15 @@
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clangxx --coverage main.cpp -o t
+// RUN: %run ./t
+// RUN: llvm-cov gcov -t t-main. | FileCheck %s
+
+//--- main.cpp
+#include <stdio.h>
+
+int main(int argc, char *argv[]) { // CHECK:      2: [[#]]:int main
+  puts("");                        // CHECK-NEXT: 2: [[#]]:
+#line 3
+  puts(""); // line 3
+  return 0; // line 4
+}
+// CHECK-NOT:  {{^ +[0-9]+:}}
diff --git a/compiler-rt/test/profile/Posix/gcov-file-change.cpp b/compiler-rt/test/profile/Posix/gcov-file-change.cpp
index 9d3bc79591f2..0cef1c3512f8 100644
--- a/compiler-rt/test/profile/Posix/gcov-file-change.cpp
+++ b/compiler-rt/test/profile/Posix/gcov-file-change.cpp
@@ -16,8 +16,8 @@ inline auto *const inl_var_main = // CHECK:      1: [[#]]:inline auto
 void foo(int x) {                 // CHECK-NEXT: 1: [[#]]:
   if (x) {                        // CHECK-NEXT: 1: [[#]]:
 #include "a.inc"
-  }
-}
+  } // CHECK:      1: [[#]]:
+} // CHECK-NEXT: 1: [[#]]:
 // CHECK-NOT:  {{^ +[0-9]+:}}
 
 int main(int argc, char *argv[]) { // CHECK:      1: [[#]]:int main
@@ -32,10 +32,8 @@ int main(int argc, char *argv[]) { // CHECK:      1: [[#]]:int main
 //--- a.h
 /// Apple targets doesn't enable -mconstructor-aliases by default and the count may be 4.
 struct A { A() { } };              // CHECK:      {{[24]}}: [[#]]:struct A
-inline auto *const inl_var_a =
-    new A;
-/// TODO a.inc:1 should have line execution.
-// CHECK-NOT:  {{^ +[0-9]+:}}
+inline auto *const inl_var_a =     // CHECK-NEXT: 1: [[#]]:
+    new A;                         // CHECK-NEXT: 1: [[#]]:
 
 //--- a.inc
-puts("");
+puts(""); // CHECK:      1: [[#]]:puts
diff --git a/llvm/include/llvm/ProfileData/GCOV.h b/llvm/include/llvm/ProfileData/GCOV.h
index 0dc33d062e4f..bc407182a094 100644
--- a/llvm/include/llvm/ProfileData/GCOV.h
+++ b/llvm/include/llvm/ProfileData/GCOV.h
@@ -271,6 +271,14 @@ public:
   DenseSet<const GCOVBlock *> visited;
 };
 
+/// Represent file of lines same with block_location_info in gcc.
+struct GCOVBlockLocation {
+  GCOVBlockLocation(unsigned idx) : srcIdx(idx) {}
+
+  unsigned srcIdx;
+  SmallVector<uint32_t, 4> lines;
+};
+
 /// GCOVBlock - Collects block information.
 class GCOVBlock {
 public:
@@ -281,8 +289,13 @@ public:
 
   GCOVBlock(uint32_t N) : number(N) {}
 
-  void addLine(uint32_t N) { lines.push_back(N); }
-  uint32_t getLastLine() const { return lines.back(); }
+  void addLine(uint32_t N) {
+    locations.back().lines.push_back(N);
+    lastLine = N;
+  }
+  void addFile(unsigned fileIdx) { locations.emplace_back(fileIdx); }
+
+  uint32_t getLastLine() const { return lastLine; }
   uint64_t getCount() const { return count; }
 
   void addSrcEdge(GCOVArc *Edge) { pred.push_back(Edge); }
@@ -311,7 +324,8 @@ public:
   uint64_t count = 0;
   SmallVector<GCOVArc *, 2> pred;
   SmallVector<GCOVArc *, 2> succ;
-  SmallVector<uint32_t, 4> lines;
+  SmallVector<GCOVBlockLocation> locations;
+  uint32_t lastLine = 0;
   bool traversable = false;
   GCOVArc *incoming = nullptr;
 };
diff --git a/llvm/lib/ProfileData/GCOV.cpp b/llvm/lib/ProfileData/GCOV.cpp
index ecb12c045b5b..7d0a243d0238 100644
--- a/llvm/lib/ProfileData/GCOV.cpp
+++ b/llvm/lib/ProfileData/GCOV.cpp
@@ -191,7 +191,7 @@ bool GCOVFile::readGCNO(GCOVBuffer &buf) {
           buf.readString(filename);
           if (filename.empty())
             break;
-          // TODO Unhandled
+          Block.addFile(addNormalizedPathToMap(filename));
         }
       }
     }
@@ -456,11 +456,13 @@ void GCOVBlock::print(raw_ostream &OS) const {
     }
     OS << "\n";
   }
-  if (!lines.empty()) {
-    OS << "\tLines : ";
-    for (uint32_t N : lines)
-      OS << (N) << ",";
-    OS << "\n";
+  if (!locations.empty()) {
+    for (const GCOVBlockLocation &loc : locations) {
+      OS << "\tFile: " << loc.srcIdx << ": ";
+      for (uint32_t N : loc.lines)
+        OS << (N) << ",";
+      OS << "\n";
+    }
   }
 }
 
@@ -701,20 +703,25 @@ void Context::collectFunction(GCOVFunction &f, Summary &summary) {
   SmallSet<uint32_t, 16> lines;
   SmallSet<uint32_t, 16> linesExec;
   for (const GCOVBlock &b : f.blocksRange()) {
-    if (b.lines.empty())
+    if (b.locations.empty())
       continue;
-    uint32_t maxLineNum = *llvm::max_element(b.lines);
-    if (maxLineNum >= si.lines.size())
-      si.lines.resize(maxLineNum + 1);
-    for (uint32_t lineNum : b.lines) {
-      LineInfo &line = si.lines[lineNum];
-      if (lines.insert(lineNum).second)
-        ++summary.lines;
-      if (b.count && linesExec.insert(lineNum).second)
-        ++summary.linesExec;
-      line.exists = true;
-      line.count += b.count;
-      line.blocks.push_back(&b);
+    for (const GCOVBlockLocation &loc : b.locations) {
+      SourceInfo &locSource = sources[loc.srcIdx];
+      uint32_t maxLineNum = *llvm::max_element(loc.lines);
+      if (maxLineNum >= locSource.lines.size())
+        locSource.lines.resize(maxLineNum + 1);
+      for (uint32_t lineNum : loc.lines) {
+        LineInfo &line = locSource.lines[lineNum];
+        line.exists = true;
+        line.count += b.count;
+        line.blocks.push_back(&b);
+        if (f.srcIdx == loc.srcIdx) {
+          if (lines.insert(lineNum).second)
+            ++summary.lines;
+          if (b.count && linesExec.insert(lineNum).second)
+            ++summary.linesExec;
+        }
+      }
     }
   }
 }
diff --git a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 3dfb36f4f181..4c7b8c69c1bf 100644
--- a/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -210,12 +210,12 @@ static StringRef getFunctionName(const DISubprogram *SP) {
   return SP->getName();
 }
 
-/// Extract a filename for a DISubprogram.
+/// Extract a filename for a DIScope.
 ///
 /// Prefer relative paths in the coverage notes. Clang also may split
 /// up absolute paths into a directory and filename component. When
 /// the relative path doesn't exist, reconstruct the absolute path.
-static SmallString<128> getFilename(const DISubprogram *SP) {
+static SmallString<128> getFilename(const DIScope *SP) {
   SmallString<128> Path;
   StringRef RelPath = SP->getFilename();
   if (sys::fs::exists(RelPath))
@@ -244,7 +244,9 @@ namespace {
   // list of line numbers and a single filename, representing lines that belong
   // to the block.
   class GCOVLines : public GCOVRecord {
-   public:
+  public:
+    const StringRef getFilename() { return Filename; }
+
     void addLine(uint32_t Line) {
       assert(Line != 0 && "Line zero is not a valid real line number.");
       Lines.push_back(Line);
@@ -276,7 +278,9 @@ namespace {
   class GCOVBlock : public GCOVRecord {
    public:
     GCOVLines &getFile(StringRef Filename) {
-      return LinesByFile.try_emplace(Filename, P, Filename).first->second;
+      if (Lines.empty() || Lines.back().getFilename() != Filename)
+        Lines.emplace_back(P, Filename);
+      return Lines.back();
     }
 
     void addEdge(GCOVBlock &Successor, uint32_t Flags) {
@@ -285,22 +289,16 @@ namespace {
 
     void writeOut() {
       uint32_t Len = 3;
-      SmallVector<StringMapEntry<GCOVLines> *, 32> SortedLinesByFile;
-      for (auto &I : LinesByFile) {
-        Len += I.second.length();
-        SortedLinesByFile.push_back(&I);
-      }
+
+      for (auto &L : Lines)
+        Len += L.length();
 
       write(GCOV_TAG_LINES);
       write(Len);
       write(Number);
 
-      llvm::sort(SortedLinesByFile, [](StringMapEntry<GCOVLines> *LHS,
-                                       StringMapEntry<GCOVLines> *RHS) {
-        return LHS->getKey() < RHS->getKey();
-      });
-      for (auto &I : SortedLinesByFile)
-        I->getValue().writeOut();
+      for (auto &L : Lines)
+        L.writeOut();
       write(0);
       write(0);
     }
@@ -309,7 +307,7 @@ namespace {
       // Only allow copy before edges and lines have been added. After that,
       // there are inter-block pointers (eg: edges) that won't take kindly to
       // blocks being copied or moved around.
-      assert(LinesByFile.empty());
+      assert(Lines.empty());
       assert(OutEdges.empty());
     }
 
@@ -322,7 +320,7 @@ namespace {
     GCOVBlock(GCOVProfiler *P, uint32_t Number)
         : GCOVRecord(P), Number(Number) {}
 
-    StringMap<GCOVLines> LinesByFile;
+    SmallVector<GCOVLines> Lines;
   };
 
   // A function has a unique identifier, a checksum (we leave as zero) and a
@@ -881,11 +879,10 @@ bool GCOVProfiler::emitProfileNotes(
           if (Line == Loc.getLine()) continue;
           Line = Loc.getLine();
           MDNode *Scope = Loc.getScope();
-          // TODO: Handle blocks from another file due to #line, #include, etc.
-          if (isa<DILexicalBlockFile>(Scope) || SP != getDISubprogram(Scope))
+          if (SP != getDISubprogram(Scope))
             continue;
 
-          GCOVLines &Lines = Block.getFile(Filename);
+          GCOVLines &Lines = Block.getFile(getFilename(Loc->getScope()));
           Lines.addLine(Loc.getLine());
         }
         Line = 0;
-- 
cgit v1.2.3


From 874773635d31501ab21812c05c44caf281c1acc7 Mon Sep 17 00:00:00 2001
From: Sander de Smalen <sander.desmalen@arm.com>
Date: Fri, 20 Jun 2025 10:00:55 +0100
Subject: [SLP] NFC: Simplify CandidateVFs initialization (#144882)

Also adds a comment to clarify the meaning of MaxRegVF.
---
 llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5eef2497cf90..1141c1b2babb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -21198,7 +21198,11 @@ bool SLPVectorizerPass::vectorizeStores(
         }
       }
 
+      // MaxRegVF represents the number of instructions (scalar, or vector in
+      // case of revec) that can be vectorized to naturally fit in a vector
+      // register.
       unsigned MaxRegVF = MaxVF;
+
       MaxVF = std::min<unsigned>(MaxVF, bit_floor(Operands.size()));
       if (MaxVF < MinVF) {
         LLVM_DEBUG(dbgs() << "SLP: Vectorization infeasible as MaxVF (" << MaxVF
@@ -21207,13 +21211,11 @@ bool SLPVectorizerPass::vectorizeStores(
         continue;
       }
 
-      unsigned Sz = 1 + Log2_32(MaxVF) - Log2_32(MinVF);
-      SmallVector<unsigned> CandidateVFs(Sz + (NonPowerOf2VF > 0 ? 1 : 0));
-      unsigned Size = MinVF;
-      for (unsigned &VF : reverse(CandidateVFs)) {
-        VF = Size > MaxVF ? NonPowerOf2VF : Size;
-        Size *= 2;
-      }
+      SmallVector<unsigned> CandidateVFs;
+      for (unsigned VF = std::max(MaxVF, NonPowerOf2VF); VF >= MinVF;
+           VF = divideCeil(VF, 2))
+        CandidateVFs.push_back(VF);
+
       unsigned End = Operands.size();
       unsigned Repeat = 0;
       constexpr unsigned MaxAttempts = 4;
-- 
cgit v1.2.3


From 14e89b061fdecedcec4bb035060a56588610cb5c Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Fri, 20 Jun 2025 17:01:35 +0800
Subject: [C++20] [Modules] Add exported modules as transitive imported modules

Close https://github.com/llvm/llvm-project/issues/144230

The root cause of the problem is, when we decide the transitive imports,
we didn't deal with exported imports.
---
 clang/lib/Sema/SemaModule.cpp    | 13 +++++++++----
 clang/test/Modules/pr144230.cppm | 26 ++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 4 deletions(-)
 create mode 100644 clang/test/Modules/pr144230.cppm

diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 9fcaad48d305..54ee0486763b 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -136,6 +136,7 @@ makeTransitiveImportsVisible(ASTContext &Ctx, VisibleModuleSet &VisibleModules,
          "modules only.");
 
   llvm::SmallVector<Module *, 4> Worklist;
+  llvm::SmallSet<Module *, 16> Visited;
   Worklist.push_back(Imported);
 
   Module *FoundPrimaryModuleInterface =
@@ -144,18 +145,22 @@ makeTransitiveImportsVisible(ASTContext &Ctx, VisibleModuleSet &VisibleModules,
   while (!Worklist.empty()) {
     Module *Importing = Worklist.pop_back_val();
 
-    if (VisibleModules.isVisible(Importing))
+    if (Visited.count(Importing))
       continue;
+    Visited.insert(Importing);
 
     // FIXME: The ImportLoc here is not meaningful. It may be problematic if we
     // use the sourcelocation loaded from the visible modules.
     VisibleModules.setVisible(Importing, ImportLoc);
 
     if (isImportingModuleUnitFromSameModule(Ctx, Importing, CurrentModule,
-                                            FoundPrimaryModuleInterface))
+                                            FoundPrimaryModuleInterface)) {
       for (Module *TransImported : Importing->Imports)
-        if (!VisibleModules.isVisible(TransImported))
-          Worklist.push_back(TransImported);
+        Worklist.push_back(TransImported);
+
+      for (auto [Exports, _] : Importing->Exports)
+        Worklist.push_back(Exports);
+    }
   }
 }
 
diff --git a/clang/test/Modules/pr144230.cppm b/clang/test/Modules/pr144230.cppm
new file mode 100644
index 000000000000..7de9fc6461ab
--- /dev/null
+++ b/clang/test/Modules/pr144230.cppm
@@ -0,0 +1,26 @@
+// RUN: rm -rf %t
+// RUN: mkdir -p %t
+// RUN: split-file %s %t
+
+// RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-module-interface -o %t/A.pcm
+// RUN: %clang_cc1 -std=c++20 %t/P.cppm -emit-module-interface -o %t/M-P.pcm -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 %t/M.cppm -emit-module-interface -o %t/M.pcm -fprebuilt-module-path=%t
+// RUN: %clang_cc1 -std=c++20 %t/M.cpp -fprebuilt-module-path=%t -fsyntax-only -verify
+
+//--- A.cppm
+export module A;
+export using T = int;
+
+//--- P.cppm
+export module M:P;
+import A;
+
+//--- M.cppm
+export module M;
+export import :P;
+
+//--- M.cpp
+// expected-no-diagnostics
+module M;
+
+T x;
-- 
cgit v1.2.3


From f704782c63ac1d567422f66072bebd49c41723f9 Mon Sep 17 00:00:00 2001
From: Matthew Devereau <matthew.devereau@arm.com>
Date: Fri, 20 Jun 2025 10:07:48 +0100
Subject: [AArch64][SelectionDAG] Fix UDOT regression (#144907)

Fix broken check in AArch64ISelLowering for bailing from ZExt
optimizations when there is a partial reduction intrinsic.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    | 16 ++---
 .../AArch64/neon-partial-reduce-dot-product.ll     | 80 ++++++++++++++--------
 2 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 69f8c6c21a55..8f208c4d006c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16869,14 +16869,14 @@ bool AArch64TargetLowering::optimizeExtendOrTruncateConversion(
     if (SrcWidth * 4 <= DstWidth) {
       if (all_of(I->users(), [&](auto *U) {
             auto *SingleUser = cast<Instruction>(&*U);
-            return (
-                match(SingleUser, m_c_Mul(m_Specific(I), m_SExt(m_Value()))) ||
-                (match(SingleUser,
-                       m_Intrinsic<
-                           Intrinsic::experimental_vector_partial_reduce_add>(
-                           m_Value(), m_Specific(I))) &&
-                 !shouldExpandPartialReductionIntrinsic(
-                     cast<IntrinsicInst>(SingleUser))));
+            if (match(SingleUser, m_c_Mul(m_Specific(I), m_SExt(m_Value()))))
+              return true;
+            if (match(SingleUser,
+                      m_Intrinsic<
+                          Intrinsic::experimental_vector_partial_reduce_add>(
+                          m_Value(), m_Specific(I))))
+              return true;
+            return false;
           }))
         return false;
     }
diff --git a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
index 0ea80a075fae..0db4dc07fbfe 100644
--- a/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
+++ b/llvm/test/CodeGen/AArch64/neon-partial-reduce-dot-product.ll
@@ -772,35 +772,57 @@ define <4 x i32> @udot_no_bin_op(<4 x i32> %acc, <16 x i8> %a){
 }
 
 define <4 x i32> @udot_no_bin_op_in_loop(ptr %p){
-; CHECK-COMMON-LABEL: udot_no_bin_op_in_loop:
-; CHECK-COMMON:       // %bb.0: // %entry
-; CHECK-COMMON-NEXT:    adrp x8, .LCPI16_0
-; CHECK-COMMON-NEXT:    movi v2.2d, #0000000000000000
-; CHECK-COMMON-NEXT:    adrp x9, .LCPI16_2
-; CHECK-COMMON-NEXT:    ldr q1, [x8, :lo12:.LCPI16_0]
-; CHECK-COMMON-NEXT:    adrp x8, .LCPI16_1
-; CHECK-COMMON-NEXT:    adrp x10, .LCPI16_3
-; CHECK-COMMON-NEXT:    ldr q3, [x8, :lo12:.LCPI16_1]
-; CHECK-COMMON-NEXT:    ldr q4, [x9, :lo12:.LCPI16_2]
-; CHECK-COMMON-NEXT:    ldr q5, [x10, :lo12:.LCPI16_3]
-; CHECK-COMMON-NEXT:    mov x8, xzr
-; CHECK-COMMON-NEXT:  .LBB16_1: // %vector.body
-; CHECK-COMMON-NEXT:    // =>This Inner Loop Header: Depth=1
-; CHECK-COMMON-NEXT:    ldr q6, [x0, x8]
-; CHECK-COMMON-NEXT:    mov v0.16b, v2.16b
-; CHECK-COMMON-NEXT:    add x8, x8, #16
-; CHECK-COMMON-NEXT:    cmp x8, #16
-; CHECK-COMMON-NEXT:    tbl v7.16b, { v6.16b }, v3.16b
-; CHECK-COMMON-NEXT:    tbl v16.16b, { v6.16b }, v4.16b
-; CHECK-COMMON-NEXT:    tbl v17.16b, { v6.16b }, v5.16b
-; CHECK-COMMON-NEXT:    tbl v6.16b, { v6.16b }, v1.16b
-; CHECK-COMMON-NEXT:    add v2.4s, v2.4s, v17.4s
-; CHECK-COMMON-NEXT:    add v7.4s, v16.4s, v7.4s
-; CHECK-COMMON-NEXT:    add v2.4s, v2.4s, v7.4s
-; CHECK-COMMON-NEXT:    add v2.4s, v2.4s, v6.4s
-; CHECK-COMMON-NEXT:    b.ne .LBB16_1
-; CHECK-COMMON-NEXT:  // %bb.2: // %end
-; CHECK-COMMON-NEXT:    ret
+; CHECK-NODOT-LABEL: udot_no_bin_op_in_loop:
+; CHECK-NODOT:       // %bb.0: // %entry
+; CHECK-NODOT-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-NODOT-NEXT:    mov x8, xzr
+; CHECK-NODOT-NEXT:  .LBB16_1: // %vector.body
+; CHECK-NODOT-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-NODOT-NEXT:    ldr q2, [x0, x8]
+; CHECK-NODOT-NEXT:    mov v0.16b, v1.16b
+; CHECK-NODOT-NEXT:    add x8, x8, #16
+; CHECK-NODOT-NEXT:    cmp x8, #16
+; CHECK-NODOT-NEXT:    ushll v3.8h, v2.8b, #0
+; CHECK-NODOT-NEXT:    ushll2 v2.8h, v2.16b, #0
+; CHECK-NODOT-NEXT:    uaddw v1.4s, v1.4s, v3.4h
+; CHECK-NODOT-NEXT:    uaddw2 v1.4s, v1.4s, v3.8h
+; CHECK-NODOT-NEXT:    uaddw v1.4s, v1.4s, v2.4h
+; CHECK-NODOT-NEXT:    uaddw2 v1.4s, v1.4s, v2.8h
+; CHECK-NODOT-NEXT:    b.ne .LBB16_1
+; CHECK-NODOT-NEXT:  // %bb.2: // %end
+; CHECK-NODOT-NEXT:    ret
+;
+; CHECK-DOT-LABEL: udot_no_bin_op_in_loop:
+; CHECK-DOT:       // %bb.0: // %entry
+; CHECK-DOT-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-DOT-NEXT:    movi v2.16b, #1
+; CHECK-DOT-NEXT:    mov x8, xzr
+; CHECK-DOT-NEXT:  .LBB16_1: // %vector.body
+; CHECK-DOT-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-NEXT:    ldr q3, [x0, x8]
+; CHECK-DOT-NEXT:    mov v0.16b, v1.16b
+; CHECK-DOT-NEXT:    add x8, x8, #16
+; CHECK-DOT-NEXT:    cmp x8, #16
+; CHECK-DOT-NEXT:    udot v1.4s, v3.16b, v2.16b
+; CHECK-DOT-NEXT:    b.ne .LBB16_1
+; CHECK-DOT-NEXT:  // %bb.2: // %end
+; CHECK-DOT-NEXT:    ret
+;
+; CHECK-DOT-I8MM-LABEL: udot_no_bin_op_in_loop:
+; CHECK-DOT-I8MM:       // %bb.0: // %entry
+; CHECK-DOT-I8MM-NEXT:    movi v1.2d, #0000000000000000
+; CHECK-DOT-I8MM-NEXT:    movi v2.16b, #1
+; CHECK-DOT-I8MM-NEXT:    mov x8, xzr
+; CHECK-DOT-I8MM-NEXT:  .LBB16_1: // %vector.body
+; CHECK-DOT-I8MM-NEXT:    // =>This Inner Loop Header: Depth=1
+; CHECK-DOT-I8MM-NEXT:    ldr q3, [x0, x8]
+; CHECK-DOT-I8MM-NEXT:    mov v0.16b, v1.16b
+; CHECK-DOT-I8MM-NEXT:    add x8, x8, #16
+; CHECK-DOT-I8MM-NEXT:    cmp x8, #16
+; CHECK-DOT-I8MM-NEXT:    udot v1.4s, v3.16b, v2.16b
+; CHECK-DOT-I8MM-NEXT:    b.ne .LBB16_1
+; CHECK-DOT-I8MM-NEXT:  // %bb.2: // %end
+; CHECK-DOT-I8MM-NEXT:    ret
 entry:
   br label %vector.body
 
-- 
cgit v1.2.3


From b85387dfe8e67ee8a142a1faf25325761c343577 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 20 Jun 2025 10:39:06 +0200
Subject: [SCCP] Check instruction type before querying PredicateInfo (NFC)

Do the cheap intrinsic check before the hash lookup for the
PredicateInfo.
---
 llvm/lib/Transforms/Utils/SCCPSolver.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index cc0bb4735c23..e1b1cf68cd9f 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -771,9 +771,9 @@ public:
 
     for (BasicBlock &BB : F) {
       for (Instruction &Inst : llvm::make_early_inc_range(BB)) {
-        if (It->second->getPredicateInfoFor(&Inst)) {
-          if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
-            if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+        if (auto *II = dyn_cast<IntrinsicInst>(&Inst)) {
+          if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
+            if (It->second->getPredicateInfoFor(&Inst)) {
               Value *Op = II->getOperand(0);
               Inst.replaceAllUsesWith(Op);
               Inst.eraseFromParent();
-- 
cgit v1.2.3


From eb0f1dc00e5d0e591fe912c1aaf9dd9d01d94b8d Mon Sep 17 00:00:00 2001
From: Peter Smith <peter.smith@arm.com>
Date: Fri, 20 Jun 2025 10:11:42 +0100
Subject: [LLD][ELF] Include offset when adding Thunk symbols (#144995)

Include the offset of a thunk in the ThunkSection when adding symbols.

At Thunk creation time the offset is set to 0 as we don't know where in
the ThunkSection the Thunk will end up. The symbol values are updated by
the setOffset() call in assignOffsets().

When we transform a thunk from a short to a long, we sometimes add a
mapping symbol. At this point the offset of the thunk is non zero and we
need to account for that when defining the symbol, as the setOffset()
call subtracts the offset before adding the new one back in.

To test; added a second thunk that is converted to a long thunk to
aarch64-thunk-bit-multipass. This second thunk is given a non zero
offset from the start of the Thunk Section so we can observe the mapping
symbol being put in the wrong place without accounting for the offset.

fixes: https://github.com/llvm/llvm-project/issues/142326
---
 lld/ELF/Thunks.cpp                         |  3 ++-
 lld/test/ELF/aarch64-thunk-bti-multipass.s | 22 +++++++++++++++++++++-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index bad1b4b85735..c26ba76bccb7 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -601,7 +601,8 @@ public:
 
 Defined *Thunk::addSymbol(StringRef name, uint8_t type, uint64_t value,
                           InputSectionBase &section) {
-  Defined *d = addSyntheticLocal(ctx, name, type, value, /*size=*/0, section);
+  Defined *d =
+      addSyntheticLocal(ctx, name, type, value + offset, /*size=*/0, section);
   syms.push_back(d);
   return d;
 }
diff --git a/lld/test/ELF/aarch64-thunk-bti-multipass.s b/lld/test/ELF/aarch64-thunk-bti-multipass.s
index f2ff914fb850..f470d832843b 100644
--- a/lld/test/ELF/aarch64-thunk-bti-multipass.s
+++ b/lld/test/ELF/aarch64-thunk-bti-multipass.s
@@ -29,6 +29,7 @@
 _start:
 /// Call that requires a thunk.
  bl fn1
+ bl fn2
 /// padding so that the thunk for fn1 is placed after this section is
 /// sufficiently close to the target to be within short range, but only
 /// just so that a small displacement will mean a long thunk is needed.
@@ -39,6 +40,7 @@ _start:
 
 // CHECK-LABEL: <_start>:
 // CHECK-NEXT: 10001000: bl  0x10002008 <__AArch64AbsLongThunk_fn1>
+// CHECK-NEXT:           bl  0x10002018 <__AArch64AbsLongThunk_fn2>
 
 // CHECK-LABEL: <__AArch64AbsLongThunk_fn1>:
 // CHECK-NEXT: 10002008: ldr     x16, 0x10002010 <__AArch64AbsLongThunk_fn1+0x8>
@@ -46,6 +48,12 @@ _start:
 // CHECK-NEXT:           00 30 00 18    .word   0x18003000
 // CHECK-NEXT:           00 00 00 00    .word   0x00000000
 
+// CHECK-LABEL: <__AArch64AbsLongThunk_fn2>:
+// CHECK-NEXT: 10002018: ldr     x16, 0x10002020 <__AArch64AbsLongThunk_fn2+0x8>
+// CHECK-NEXT:           br      x16
+// CHECK-NEXT:           04 40 00 18    .word   0x18004004
+// CHECK-NEXT:           00 00 00 00    .word   0x00000000
+
 .section .text.1, "ax", %progbits
 .balign 0x1000
 .global farcall
@@ -75,6 +83,12 @@ farcall:
 fn1:
  ret
 
+.section .text.3, "ax", %progbits
+.global fn2
+.type fn2, %function
+fn2:
+ ret
+
 .section .text.far, "ax", %progbits
 .type far, %function
 .global far
@@ -88,6 +102,12 @@ far:
 // CHECK-LABEL: <fn1>:
 // CHECK-NEXT: 18004000: ret
 
+// CHECK-LABEL: <__AArch64BTIThunk_fn2>:
+// CHECK-NEXT: 18004004:       bti     c
+
+// CHECK-LABEL: <fn2>:
+// CHECK-NEXT: 18004008:       ret
+
 // CHECK-LABEL: <__AArch64BTIThunk_far>:
 // CHECK-NEXT: 30000000: bti     c
 
@@ -104,6 +124,6 @@ SECTIONS {
   .rodata 0x10000000 : { *(.note.gnu.property) } :low
   .text_low : { *(.text.0) } :low
   .text 0x18001000 : { *(.text.1) } :mid
-  .text_aligned : { *(.text.2) } :mid
+  .text_aligned : { *(.text.2) *(.text.3) } :mid
   .text_high 0x30000000 : { *(.text.far) } :high
 }
-- 
cgit v1.2.3


From 61972054f3fcaf59096799342bac9c93dd9aa432 Mon Sep 17 00:00:00 2001
From: Alexis Engelke <engelke@in.tum.de>
Date: Fri, 20 Jun 2025 11:23:00 +0200
Subject: [CodeGen] Limit number of analyzed predecessors

MachineBlockPlacement has quadratic runtime in the number of
predecessors: in some situation, for an edge, all predecessors of the
successor are considered.

Limit the number of considered predecessors to bound compile time for
large functions.

Pull Request: https://github.com/llvm/llvm-project/pull/142584
---
 llvm/lib/CodeGen/MachineBlockPlacement.cpp | 16 ++++++++++
 llvm/test/CodeGen/RISCV/branch.ll          | 49 ++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/llvm/lib/CodeGen/MachineBlockPlacement.cpp b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
index 08fe3d47e2ff..2dbabfe345d5 100644
--- a/llvm/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/llvm/lib/CodeGen/MachineBlockPlacement.cpp
@@ -104,6 +104,12 @@ static cl::opt<unsigned> MaxBytesForAlignmentOverride(
              "alignment"),
     cl::init(0), cl::Hidden);
 
+static cl::opt<unsigned> PredecessorLimit(
+    "block-placement-predecessor-limit",
+    cl::desc("For blocks with more predecessors, certain layout optimizations"
+             "will be disabled to prevent quadratic compile time."),
+    cl::init(1000), cl::Hidden);
+
 // FIXME: Find a good default for this flag and remove the flag.
 static cl::opt<unsigned> ExitBlockBias(
     "block-placement-exit-block-bias",
@@ -1030,6 +1036,11 @@ bool MachineBlockPlacement::isTrellis(
   SmallPtrSet<const MachineBasicBlock *, 8> SeenPreds;
 
   for (MachineBasicBlock *Succ : ViableSuccs) {
+    // Compile-time optimization: runtime is quadratic in the number of
+    // predecessors. For such uncommon cases, exit early.
+    if (Succ->pred_size() > PredecessorLimit)
+      return false;
+
     int PredCount = 0;
     for (auto *SuccPred : Succ->predecessors()) {
       // Allow triangle successors, but don't count them.
@@ -1472,6 +1483,11 @@ bool MachineBlockPlacement::hasBetterLayoutPredecessor(
   if (SuccChain.UnscheduledPredecessors == 0)
     return false;
 
+  // Compile-time optimization: runtime is quadratic in the number of
+  // predecessors. For such uncommon cases, exit early.
+  if (Succ->pred_size() > PredecessorLimit)
+    return false;
+
   // There are two basic scenarios here:
   // -------------------------------------
   // Case 1: triangular shape CFG (if-then):
diff --git a/llvm/test/CodeGen/RISCV/branch.ll b/llvm/test/CodeGen/RISCV/branch.ll
index 578080cd3a24..ed86ca8ca4dd 100644
--- a/llvm/test/CodeGen/RISCV/branch.ll
+++ b/llvm/test/CodeGen/RISCV/branch.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=riscv32 -verify-machineinstrs < %s \
 ; RUN:   | FileCheck -check-prefix=RV32I %s
+; RUN: llc -mtriple=riscv32 -verify-machineinstrs -block-placement-predecessor-limit=10 < %s \
+; RUN:   | FileCheck -check-prefix=RV32I-MBPLIMIT %s
 
 define void @foo(i32 %a, ptr %b, i1 %c) nounwind {
 ; RV32I-LABEL: foo:
@@ -48,6 +50,53 @@ define void @foo(i32 %a, ptr %b, i1 %c) nounwind {
 ; RV32I-NEXT:    lw zero, 0(a1)
 ; RV32I-NEXT:  .LBB0_14: # %end
 ; RV32I-NEXT:    ret
+;
+; RV32I-MBPLIMIT-LABEL: foo:
+; RV32I-MBPLIMIT:       # %bb.0:
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bne a3, a0, .LBB0_2
+; RV32I-MBPLIMIT-NEXT:  .LBB0_1: # %end
+; RV32I-MBPLIMIT-NEXT:    ret
+; RV32I-MBPLIMIT-NEXT:  .LBB0_2: # %test2
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bne a3, a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.3: # %test3
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    blt a3, a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.4: # %test4
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bge a3, a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.5: # %test5
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bltu a3, a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.6: # %test6
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bgeu a3, a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.7: # %test7
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    blt a0, a3, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.8: # %test8
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bge a0, a3, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.9: # %test9
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bltu a0, a3, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.10: # %test10
+; RV32I-MBPLIMIT-NEXT:    lw a3, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bgeu a0, a3, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.11: # %test11
+; RV32I-MBPLIMIT-NEXT:    lw zero, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    andi a2, a2, 1
+; RV32I-MBPLIMIT-NEXT:    bnez a2, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.12: # %test12
+; RV32I-MBPLIMIT-NEXT:    lw a0, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    bgez a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.13: # %test13
+; RV32I-MBPLIMIT-NEXT:    lw a0, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    blez a0, .LBB0_1
+; RV32I-MBPLIMIT-NEXT:  # %bb.14: # %test14
+; RV32I-MBPLIMIT-NEXT:    lw zero, 0(a1)
+; RV32I-MBPLIMIT-NEXT:    ret
   %val1 = load volatile i32, ptr %b
   %tst1 = icmp eq i32 %val1, %a
   br i1 %tst1, label %end, label %test2
-- 
cgit v1.2.3


From f577516d91dc1ae5b9c8a3bcad81558bc19ccf65 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 20 Jun 2025 11:23:33 +0200
Subject: [mlir][arith] Add back ElementwiseMappable to `arith.trunci`
 (#145000)

This trait was accidentally dropped in #144863.
---
 mlir/include/mlir/Dialect/Arith/IR/ArithOps.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 993f36f556e8..0518cac156eb 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -1271,7 +1271,7 @@ def Arith_ScalingExtFOp
 // TruncIOp
 //===----------------------------------------------------------------------===//
 
-def Arith_TruncIOp : Op<Arith_Dialect, "trunci",
+def Arith_TruncIOp : Arith_Op<"trunci",
     [Pure, SameOperandsAndResultShape, SameInputOutputTensorDims,
      DeclareOpInterfaceMethods<CastOpInterface>,
      DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRanges"]>,
-- 
cgit v1.2.3


From dd4776d429bd20050c80749f669367a0574520c8 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 20 Jun 2025 18:26:56 +0900
Subject: AMDGPU: Remove AMDGPUInstrInfo class (#144984)

This was never constructed and only provided one static helper
function.
---
 llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp         | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp         | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp            | 7 +------
 llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h              | 9 ++-------
 llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp | 2 +-
 llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp     | 2 +-
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp             | 2 +-
 7 files changed, 8 insertions(+), 18 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index e52c2d7fde43..6e990cb2e160 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -3936,7 +3936,7 @@ bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
   const auto *Ld = cast<LoadSDNode>(N);
 
   const MachineMemOperand *MMO = Ld->getMemOperand();
-  if (N->isDivergent() && !AMDGPUInstrInfo::isUniformMMO(MMO))
+  if (N->isDivergent() && !AMDGPU::isUniformMMO(MMO))
     return false;
 
   return MMO->getSize().hasValue() &&
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index c51cc2a2fe52..134adc681215 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -848,7 +848,7 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(
        AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
        (isa<LoadSDNode>(N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
         MN->isInvariant())) &&
-      AMDGPUInstrInfo::isUniformMMO(MN->getMemOperand()))
+      AMDGPU::isUniformMMO(MN->getMemOperand()))
     return false;
 
   // Don't produce extloads from sub 32-bit types. SI doesn't have scalar
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
index 627ac6b0063e..5085e86d71c9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp
@@ -22,17 +22,12 @@
 
 using namespace llvm;
 
-// Pin the vtable to this file.
-//void AMDGPUInstrInfo::anchor() {}
-
-AMDGPUInstrInfo::AMDGPUInstrInfo(const GCNSubtarget &ST) { }
-
 Intrinsic::ID AMDGPU::getIntrinsicID(const MachineInstr &I) {
   return I.getOperand(I.getNumExplicitDefs()).getIntrinsicID();
 }
 
 // TODO: Should largely merge with AMDGPUTTIImpl::isSourceOfDivergence.
-bool AMDGPUInstrInfo::isUniformMMO(const MachineMemOperand *MMO) {
+bool AMDGPU::isUniformMMO(const MachineMemOperand *MMO) {
   const Value *Ptr = MMO->getValue();
   // UndefValue means this is a load of a kernel input.  These are uniform.
   // Sometimes LDS instructions have constant pointers.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
index e00c02cfdb86..0eb00cbc2f46 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.h
@@ -23,15 +23,10 @@ class GCNSubtarget;
 class MachineMemOperand;
 class MachineInstr;
 
-class AMDGPUInstrInfo {
-public:
-  explicit AMDGPUInstrInfo(const GCNSubtarget &st);
-
-  static bool isUniformMMO(const MachineMemOperand *MMO);
-};
-
 namespace AMDGPU {
 
+bool isUniformMMO(const MachineMemOperand *MMO);
+
 /// Return the intrinsic ID for opcodes with the G_AMDGPU_INTRIN_ prefix.
 ///
 /// These opcodes have an Intrinsic::ID operand similar to a GIntrinsic. But
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
index db3f6bd360b3..a60855cc4f2d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp
@@ -630,7 +630,7 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
   });
 
   Predicate isUniMMO([](const MachineInstr &MI) -> bool {
-    return AMDGPUInstrInfo::isUniformMMO(*MI.memoperands_begin());
+    return AMDGPU::isUniformMMO(*MI.memoperands_begin());
   });
 
   Predicate isConst([](const MachineInstr &MI) -> bool {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 4391a48ff2b6..dca55dafcc5e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -463,7 +463,7 @@ bool AMDGPURegisterBankInfo::isScalarLoadLegal(const MachineInstr &MI) const {
          (IsConst || !MMO->isVolatile()) &&
          // Memory must be known constant, or not written before this load.
          (IsConst || MMO->isInvariant() || (MMO->getFlags() & MONoClobber)) &&
-         AMDGPUInstrInfo::isUniformMMO(MMO);
+         AMDGPU::isUniformMMO(MMO);
 }
 
 RegisterBankInfo::InstructionMappings
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b1e77a282e41..07d79d677104 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -10866,7 +10866,7 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
       (AS == AMDGPUAS::GLOBAL_ADDRESS &&
        Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() &&
        isMemOpHasNoClobberedMemOperand(Load))) {
-    if ((!Op->isDivergent() || AMDGPUInstrInfo::isUniformMMO(MMO)) &&
+    if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) &&
         Alignment >= Align(4) && NumElements < 32) {
       if (MemVT.isPow2VectorType() ||
           (Subtarget->hasScalarDwordx3Loads() && NumElements == 3))
-- 
cgit v1.2.3


From f75973949b0e51eb4b3852c903d08bc72bf5d459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81d=C3=A1m=20Kallai?= <kadam@inf.u-szeged.hu>
Date: Fri, 20 Jun 2025 11:40:35 +0200
Subject: [BOLT][AArch64] Add support for SPE brstack format (#129231)

Since Linux 6.14, Perf gained the ability to report SPE branch events
using the `brstack` format, which matches the layout of LBR/BRBE.

This patch reuses the existing LBR parsing logic to support SPE.

Example SPE brstack format:
```bash
perf script -i perf.data -F pid,brstack --itrace=bl
```
```
  PID       FROM / TO / PREDICTED

16984  0x72e342e5f4/0x72e36192d0/M/-/-/11/RET/-
16984  0x72e7b8b3b4/0x72e7b8b3b8/PN/-/-/11/COND/-
16984  0x72e7b92b48/0x72e7b92b4c/PN/-/-/8/COND/-
16984  0x72eacc6b7c/0x760cc94b00/P/-/-/9/RET/-
16984  0x72e3f210fc/0x72e3f21068/P/-/-/4//-
16984  0x72e39b8c5c/0x72e3627b24/P/-/-/4//-
16984  0x72e7b89d20/0x72e7b92bbc/P/-/-/4/RET/-
```

SPE brstack flags can be two characters long: `PN` or `MN`:
- `P` = predicted branch
- `M` = mispredicted branch
- `N` = optionally appears when the branch is NOT-TAKEN
    - flag is relevant only to  conditional branches


Example of usage with BOLT:

1. Capture SPE branch events:
```bash
perf record -e 'arm_spe_0/branch_filter=1/u' -- binary
```

2. Convert profile for BOLT:
```bash
perf2bolt -p perf.data -o perf.fdata --spe binary
```

3. Run BOLT Optimization:
```bash
llvm-bolt binary -o binary.bolted --data   perf.fdata ...
```

A unit test verifies the parsing of the 'SPE brstack format'.

---------

Co-authored-by: Paschalis Mpeis <paschalis.mpeis@arm.com>
---
 bolt/include/bolt/Profile/DataAggregator.h     |   2 +
 bolt/include/bolt/Utils/CommandLineOpts.h      |   1 +
 bolt/lib/Profile/DataAggregator.cpp            |  61 +++++++--
 bolt/test/perf2bolt/AArch64/perf2bolt-spe.test |  12 ++
 bolt/test/perf2bolt/X86/perf2bolt-spe.test     |   9 ++
 bolt/tools/driver/llvm-bolt.cpp                |   7 ++
 bolt/unittests/Profile/CMakeLists.txt          |  14 +++
 bolt/unittests/Profile/PerfSpeEvents.cpp       | 164 +++++++++++++++++++++++++
 8 files changed, 258 insertions(+), 12 deletions(-)
 create mode 100644 bolt/test/perf2bolt/AArch64/perf2bolt-spe.test
 create mode 100644 bolt/test/perf2bolt/X86/perf2bolt-spe.test
 create mode 100644 bolt/unittests/Profile/PerfSpeEvents.cpp

diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index 96969cf53bac..662fe2a49afe 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -85,6 +85,8 @@ private:
   };
   friend raw_ostream &operator<<(raw_ostream &OS, const LBREntry &);
 
+  friend struct PerfSpeEventsTestHelper;
+
   struct PerfBranchSample {
     SmallVector<LBREntry, 32> LBR;
   };
diff --git a/bolt/include/bolt/Utils/CommandLineOpts.h b/bolt/include/bolt/Utils/CommandLineOpts.h
index 4acce5a3e832..a75b6bf720ec 100644
--- a/bolt/include/bolt/Utils/CommandLineOpts.h
+++ b/bolt/include/bolt/Utils/CommandLineOpts.h
@@ -48,6 +48,7 @@ extern llvm::cl::OptionCategory BinaryAnalysisCategory;
 extern llvm::cl::opt<unsigned> AlignText;
 extern llvm::cl::opt<unsigned> AlignFunctions;
 extern llvm::cl::opt<bool> AggregateOnly;
+extern llvm::cl::opt<bool> ArmSPE;
 extern llvm::cl::opt<unsigned> BucketsPerLine;
 extern llvm::cl::opt<bool> CompactCodeModel;
 extern llvm::cl::opt<bool> DiffOnly;
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 178c9d3a6373..e03fa9bd5322 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -49,6 +49,9 @@ static cl::opt<bool>
                      cl::desc("aggregate basic samples (without LBR info)"),
                      cl::cat(AggregatorCategory));
 
+cl::opt<bool> ArmSPE("spe", cl::desc("Enable Arm SPE mode."),
+                     cl::cat(AggregatorCategory));
+
 static cl::opt<std::string>
     ITraceAggregation("itrace",
                       cl::desc("Generate LBR info with perf itrace argument"),
@@ -181,11 +184,21 @@ void DataAggregator::start() {
 
   findPerfExecutable();
 
+  if (opts::ArmSPE) {
+    // pid    from_ip      to_ip        flags
+    // where flags could be:
+    // P/M: whether branch was Predicted or Mispredicted.
+    // N: optionally appears when the branch was Not-Taken (ie fall-through)
+    // 12345  0x123/0x456/PN/-/-/8/RET/-
+    opts::ITraceAggregation = "bl";
+    opts::ParseMemProfile = true;
+    opts::BasicAggregation = false;
+  }
+
   if (opts::BasicAggregation) {
-    launchPerfProcess("events without LBR",
-                      MainEventsPPI,
+    launchPerfProcess("events without LBR", MainEventsPPI,
                       "script -F pid,event,ip",
-                      /*Wait = */false);
+                      /*Wait = */ false);
   } else if (!opts::ITraceAggregation.empty()) {
     // Disable parsing memory profile from trace data, unless requested by user.
     if (!opts::ParseMemProfile.getNumOccurrences())
@@ -994,9 +1007,22 @@ ErrorOr<DataAggregator::LBREntry> DataAggregator::parseLBREntry() {
   if (std::error_code EC = MispredStrRes.getError())
     return EC;
   StringRef MispredStr = MispredStrRes.get();
-  if (MispredStr.size() != 1 ||
-      (MispredStr[0] != 'P' && MispredStr[0] != 'M' && MispredStr[0] != '-')) {
-    reportError("expected single char for mispred bit");
+  // SPE brstack mispredicted flags might be up to two characters long:
+  // 'PN' or 'MN'. Where 'N' optionally appears.
+  bool ValidStrSize = opts::ArmSPE
+                          ? MispredStr.size() >= 1 && MispredStr.size() <= 2
+                          : MispredStr.size() == 1;
+  bool SpeTakenBitErr =
+      (opts::ArmSPE && MispredStr.size() == 2 && MispredStr[1] != 'N');
+  bool PredictionBitErr =
+      !ValidStrSize ||
+      (MispredStr[0] != 'P' && MispredStr[0] != 'M' && MispredStr[0] != '-');
+  if (SpeTakenBitErr)
+    reportError("expected 'N' as SPE prediction bit for a not-taken branch");
+  if (PredictionBitErr)
+    reportError("expected 'P', 'M' or '-' char as a prediction bit");
+
+  if (SpeTakenBitErr || PredictionBitErr) {
     Diag << "Found: " << MispredStr << "\n";
     return make_error_code(llvm::errc::io_error);
   }
@@ -1497,7 +1523,9 @@ void DataAggregator::printBranchStacksDiagnostics(
 }
 
 std::error_code DataAggregator::parseBranchEvents() {
-  outs() << "PERF2BOLT: parse branch events...\n";
+  std::string BranchEventTypeStr =
+      opts::ArmSPE ? "SPE branch events in LBR-format" : "branch events";
+  outs() << "PERF2BOLT: parse " << BranchEventTypeStr << "...\n";
   NamedRegionTimer T("parseBranch", "Parsing branch events", TimerGroupName,
                      TimerGroupDesc, opts::TimeAggregator);
 
@@ -1525,7 +1553,8 @@ std::error_code DataAggregator::parseBranchEvents() {
     }
 
     NumEntries += Sample.LBR.size();
-    if (BAT && Sample.LBR.size() == 32 && !NeedsSkylakeFix) {
+    if (this->BC->isX86() && BAT && Sample.LBR.size() == 32 &&
+        !NeedsSkylakeFix) {
       errs() << "PERF2BOLT-WARNING: using Intel Skylake bug workaround\n";
       NeedsSkylakeFix = true;
     }
@@ -1548,10 +1577,18 @@ std::error_code DataAggregator::parseBranchEvents() {
     if (NumSamples && NumSamplesNoLBR == NumSamples) {
       // Note: we don't know if perf2bolt is being used to parse memory samples
       // at this point. In this case, it is OK to parse zero LBRs.
-      errs() << "PERF2BOLT-WARNING: all recorded samples for this binary lack "
-                "LBR. Record profile with perf record -j any or run perf2bolt "
-                "in no-LBR mode with -nl (the performance improvement in -nl "
-                "mode may be limited)\n";
+      if (!opts::ArmSPE)
+        errs()
+            << "PERF2BOLT-WARNING: all recorded samples for this binary lack "
+               "LBR. Record profile with perf record -j any or run perf2bolt "
+               "in no-LBR mode with -nl (the performance improvement in -nl "
+               "mode may be limited)\n";
+      else
+        errs()
+            << "PERF2BOLT-WARNING: All recorded samples for this binary lack "
+               "SPE brstack entries. Make sure you are running Linux perf 6.14 "
+               "or later, otherwise you get zero samples. Record the profile "
+               "with: perf record -e 'arm_spe_0/branch_filter=1/'.";
     } else {
       printBranchStacksDiagnostics(NumTotalSamples - NumSamples);
     }
diff --git a/bolt/test/perf2bolt/AArch64/perf2bolt-spe.test b/bolt/test/perf2bolt/AArch64/perf2bolt-spe.test
new file mode 100644
index 000000000000..91f5c857fbab
--- /dev/null
+++ b/bolt/test/perf2bolt/AArch64/perf2bolt-spe.test
@@ -0,0 +1,12 @@
+## Check that Arm SPE mode is available on AArch64.
+
+REQUIRES: system-linux,perf,target=aarch64{{.*}}
+
+RUN: %clang %cflags %p/../../Inputs/asm_foo.s %p/../../Inputs/asm_main.c -o %t.exe
+
+RUN: perf record -e cycles -q -o %t.perf.data -- %t.exe 2> /dev/null
+
+RUN: (perf2bolt -p %t.perf.data -o %t.perf.boltdata --spe %t.exe 2> /dev/null; exit 0) | FileCheck %s --check-prefix=CHECK-SPE-LBR
+
+CHECK-SPE-LBR: PERF2BOLT: parse SPE branch events in LBR-format
+
diff --git a/bolt/test/perf2bolt/X86/perf2bolt-spe.test b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
new file mode 100644
index 000000000000..8eed2c859509
--- /dev/null
+++ b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
@@ -0,0 +1,9 @@
+## Check that Arm SPE mode is unavailable on X86.
+
+REQUIRES: system-linux,x86_64-linux
+
+RUN: %clang %cflags %p/../../Inputs/asm_foo.s %p/../../Inputs/asm_main.c -o %t.exe
+RUN: touch %t.empty.perf.data
+RUN: not perf2bolt -p %t.empty.perf.data -o %t.perf.boltdata --spe --pa %t.exe 2>&1 | FileCheck %s
+
+CHECK: perf2bolt: -spe is available only on AArch64.
diff --git a/bolt/tools/driver/llvm-bolt.cpp b/bolt/tools/driver/llvm-bolt.cpp
index b9836c2397b6..cf1b31f8c0c6 100644
--- a/bolt/tools/driver/llvm-bolt.cpp
+++ b/bolt/tools/driver/llvm-bolt.cpp
@@ -237,6 +237,13 @@ int main(int argc, char **argv) {
       if (Error E = RIOrErr.takeError())
         report_error(opts::InputFilename, std::move(E));
       RewriteInstance &RI = *RIOrErr.get();
+
+      if (opts::AggregateOnly && !RI.getBinaryContext().isAArch64() &&
+          opts::ArmSPE) {
+        errs() << ToolName << ": -spe is available only on AArch64.\n";
+        exit(1);
+      }
+
       if (!opts::PerfData.empty()) {
         if (!opts::AggregateOnly) {
           errs() << ToolName
diff --git a/bolt/unittests/Profile/CMakeLists.txt b/bolt/unittests/Profile/CMakeLists.txt
index e0aa0926b49c..ce01c6c4b949 100644
--- a/bolt/unittests/Profile/CMakeLists.txt
+++ b/bolt/unittests/Profile/CMakeLists.txt
@@ -1,11 +1,25 @@
+set(LLVM_LINK_COMPONENTS
+  DebugInfoDWARF
+  Object
+  ${LLVM_TARGETS_TO_BUILD}
+  )
+
 add_bolt_unittest(ProfileTests
   DataAggregator.cpp
+  PerfSpeEvents.cpp
 
   DISABLE_LLVM_LINK_LLVM_DYLIB
   )
 
 target_link_libraries(ProfileTests
   PRIVATE
+  LLVMBOLTCore
   LLVMBOLTProfile
+  LLVMTargetParser
+  LLVMTestingSupport
   )
 
+foreach (tgt ${BOLT_TARGETS_TO_BUILD})
+  string(TOUPPER "${tgt}" upper)
+  target_compile_definitions(ProfileTests PRIVATE "${upper}_AVAILABLE")
+endforeach()
diff --git a/bolt/unittests/Profile/PerfSpeEvents.cpp b/bolt/unittests/Profile/PerfSpeEvents.cpp
new file mode 100644
index 000000000000..3e3e05395246
--- /dev/null
+++ b/bolt/unittests/Profile/PerfSpeEvents.cpp
@@ -0,0 +1,164 @@
+//===- bolt/unittests/Profile/PerfSpeEvents.cpp ---------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef AARCH64_AVAILABLE
+
+#include "bolt/Core/BinaryContext.h"
+#include "bolt/Profile/DataAggregator.h"
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/DebugInfo/DWARF/DWARFContext.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/TargetSelect.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace llvm::bolt;
+using namespace llvm::object;
+using namespace llvm::ELF;
+
+namespace opts {
+extern cl::opt<std::string> ReadPerfEvents;
+extern cl::opt<bool> ArmSPE;
+} // namespace opts
+
+namespace llvm {
+namespace bolt {
+
+/// Perform checks on perf SPE branch events.
+struct PerfSpeEventsTestHelper : public testing::Test {
+  void SetUp() override {
+    initalizeLLVM();
+    prepareElf();
+    initializeBOLT();
+  }
+
+protected:
+  using Trace = DataAggregator::Trace;
+  using TakenBranchInfo = DataAggregator::TakenBranchInfo;
+
+  void initalizeLLVM() {
+    llvm::InitializeAllTargetInfos();
+    llvm::InitializeAllTargetMCs();
+    llvm::InitializeAllAsmParsers();
+    llvm::InitializeAllDisassemblers();
+    llvm::InitializeAllTargets();
+    llvm::InitializeAllAsmPrinters();
+  }
+
+  void prepareElf() {
+    memcpy(ElfBuf, "\177ELF", 4);
+    ELF64LE::Ehdr *EHdr = reinterpret_cast<typename ELF64LE::Ehdr *>(ElfBuf);
+    EHdr->e_ident[llvm::ELF::EI_CLASS] = llvm::ELF::ELFCLASS64;
+    EHdr->e_ident[llvm::ELF::EI_DATA] = llvm::ELF::ELFDATA2LSB;
+    EHdr->e_machine = llvm::ELF::EM_AARCH64;
+    MemoryBufferRef Source(StringRef(ElfBuf, sizeof(ElfBuf)), "ELF");
+    ObjFile = cantFail(ObjectFile::createObjectFile(Source));
+  }
+
+  void initializeBOLT() {
+    Relocation::Arch = ObjFile->makeTriple().getArch();
+    BC = cantFail(BinaryContext::createBinaryContext(
+        ObjFile->makeTriple(), std::make_shared<orc::SymbolStringPool>(),
+        ObjFile->getFileName(), nullptr, /*IsPIC*/ false,
+        DWARFContext::create(*ObjFile.get()), {llvm::outs(), llvm::errs()}));
+    ASSERT_FALSE(!BC);
+  }
+
+  char ElfBuf[sizeof(typename ELF64LE::Ehdr)] = {};
+  std::unique_ptr<ObjectFile> ObjFile;
+  std::unique_ptr<BinaryContext> BC;
+
+  /// Helper function to export lists to show the mismatch.
+  void reportBrStackEventMismatch(
+      const std::vector<std::pair<Trace, TakenBranchInfo>> &Traces,
+      const std::vector<std::pair<Trace, TakenBranchInfo>> &ExpectedSamples) {
+    llvm::errs() << "Traces items: \n";
+    for (const auto &[Trace, BI] : Traces)
+      llvm::errs() << "{" << Trace.Branch << ", " << Trace.From << ","
+                   << Trace.To << ", " << BI.TakenCount << ", "
+                   << BI.MispredCount << "}" << "\n";
+
+    llvm::errs() << "Expected items: \n";
+    for (const auto &[Trace, BI] : ExpectedSamples)
+      llvm::errs() << "{" << Trace.Branch << ", " << Trace.From << ", "
+                   << Trace.To << ", " << BI.TakenCount << ", "
+                   << BI.MispredCount << "}" << "\n";
+  }
+
+  /// Parse and check SPE brstack as LBR.
+  void parseAndCheckBrstackEvents(
+      uint64_t PID,
+      const std::vector<std::pair<Trace, TakenBranchInfo>> &ExpectedSamples) {
+    DataAggregator DA("<pseudo input>");
+    DA.ParsingBuf = opts::ReadPerfEvents;
+    DA.BC = BC.get();
+    DataAggregator::MMapInfo MMap;
+    DA.BinaryMMapInfo.insert(std::make_pair(PID, MMap));
+
+    DA.parseBranchEvents();
+
+    EXPECT_EQ(DA.Traces.size(), ExpectedSamples.size());
+    if (DA.Traces.size() != ExpectedSamples.size())
+      reportBrStackEventMismatch(DA.Traces, ExpectedSamples);
+
+    const auto TracesBegin = DA.Traces.begin();
+    const auto TracesEnd = DA.Traces.end();
+    for (const auto &BI : ExpectedSamples) {
+      auto it = find_if(TracesBegin, TracesEnd,
+                        [&BI](const auto &Tr) { return Tr.first == BI.first; });
+
+      EXPECT_NE(it, TracesEnd);
+      EXPECT_EQ(it->second.MispredCount, BI.second.MispredCount);
+      EXPECT_EQ(it->second.TakenCount, BI.second.TakenCount);
+    }
+  }
+};
+
+} // namespace bolt
+} // namespace llvm
+
+TEST_F(PerfSpeEventsTestHelper, SpeBranchesWithBrstack) {
+  // Check perf input with SPE branch events as brstack format.
+  // Example collection command:
+  // ```
+  // perf record -e 'arm_spe_0/branch_filter=1/u' -- BINARY
+  // ```
+  // How Bolt extracts the branch events:
+  // ```
+  // perf script -F pid,brstack --itrace=bl
+  // ```
+
+  opts::ArmSPE = true;
+  opts::ReadPerfEvents = "  1234  0xa001/0xa002/PN/-/-/10/COND/-\n"
+                         "  1234  0xb001/0xb002/P/-/-/4/RET/-\n"
+                         "  1234  0xc456/0xc789/P/-/-/13/-/-\n"
+                         "  1234  0xd123/0xd456/M/-/-/7/RET/-\n"
+                         "  1234  0xe001/0xe002/P/-/-/14/RET/-\n"
+                         "  1234  0xd123/0xd456/M/-/-/7/RET/-\n"
+                         "  1234  0xf001/0xf002/MN/-/-/8/COND/-\n"
+                         "  1234  0xc456/0xc789/M/-/-/13/-/-\n";
+
+  // ExpectedSamples contains the aggregated information about
+  // a branch {{Branch From, To}, {TakenCount, MispredCount}}.
+  // Consider this example trace: {{0xd123, 0xd456, Trace::BR_ONLY},
+  // {2,2}}. This entry has a TakenCount = 2, as we have two samples for
+  // (0xd123, 0xd456) in our input. It also has MispredsCount = 2,
+  // as 'M' misprediction flag appears in both cases. BR_ONLY means
+  // the trace only contains branch data.
+  std::vector<std::pair<Trace, TakenBranchInfo>> ExpectedSamples = {
+      {{0xa001, 0xa002, Trace::BR_ONLY}, {1, 0}},
+      {{0xb001, 0xb002, Trace::BR_ONLY}, {1, 0}},
+      {{0xc456, 0xc789, Trace::BR_ONLY}, {2, 1}},
+      {{0xd123, 0xd456, Trace::BR_ONLY}, {2, 2}},
+      {{0xe001, 0xe002, Trace::BR_ONLY}, {1, 0}},
+      {{0xf001, 0xf002, Trace::BR_ONLY}, {1, 1}}};
+
+  parseAndCheckBrstackEvents(1234, ExpectedSamples);
+}
+
+#endif
-- 
cgit v1.2.3


From 238abf8ba8233ec0f2dab57a3bacbd192e78f8b6 Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 20 Jun 2025 09:40:52 +0000
Subject: [gn build] Port f75973949b0e

---
 llvm/utils/gn/secondary/bolt/unittests/Profile/BUILD.gn | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/bolt/unittests/Profile/BUILD.gn b/llvm/utils/gn/secondary/bolt/unittests/Profile/BUILD.gn
index b3f7216af059..8ac57eea6a46 100644
--- a/llvm/utils/gn/secondary/bolt/unittests/Profile/BUILD.gn
+++ b/llvm/utils/gn/secondary/bolt/unittests/Profile/BUILD.gn
@@ -3,5 +3,8 @@ import("//third-party/unittest/unittest.gni")
 unittest("ProfileTests") {
   configs += [ "//llvm/utils/gn/build:bolt_code" ]
   deps = [ "//bolt/lib/Profile" ]
-  sources = [ "DataAggregator.cpp" ]
+  sources = [
+    "DataAggregator.cpp",
+    "PerfSpeEvents.cpp",
+  ]
 }
-- 
cgit v1.2.3


From 95bd05d7cae327e431ccdaf0a452a0573ade5357 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 20 Jun 2025 12:04:04 +0200
Subject: [mlir][Func][NFC] Simplify implementation after #144706 (#145006)

---
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index c6bcb593eaad..538016927256 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -273,7 +273,7 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
 static void restoreByValRefArgumentType(
     ConversionPatternRewriter &rewriter, const LLVMTypeConverter &typeConverter,
     ArrayRef<std::optional<NamedAttribute>> byValRefNonPtrAttrs,
-    ArrayRef<BlockArgument> oldBlockArgs, LLVM::LLVMFuncOp funcOp) {
+    LLVM::LLVMFuncOp funcOp) {
   // Nothing to do for function declarations.
   if (funcOp.isExternal())
     return;
@@ -281,8 +281,8 @@ static void restoreByValRefArgumentType(
   ConversionPatternRewriter::InsertionGuard guard(rewriter);
   rewriter.setInsertionPointToStart(&funcOp.getFunctionBody().front());
 
-  for (const auto &[arg, oldArg, byValRefAttr] :
-       llvm::zip(funcOp.getArguments(), oldBlockArgs, byValRefNonPtrAttrs)) {
+  for (const auto &[arg, byValRefAttr] :
+       llvm::zip(funcOp.getArguments(), byValRefNonPtrAttrs)) {
     // Skip argument if no `llvm.byval` or `llvm.byref` attribute.
     if (!byValRefAttr)
       continue;
@@ -309,10 +309,6 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
     return rewriter.notifyMatchFailure(
         funcOp, "Only support FunctionOpInterface with FunctionType");
 
-  // Keep track of the entry block arguments. They will be needed later.
-  SmallVector<BlockArgument> oldBlockArgs =
-      llvm::to_vector(funcOp.getArguments());
-
   // Convert the original function arguments. They are converted using the
   // LLVMTypeConverter provided to this legalization pattern.
   auto varargsAttr = funcOp->getAttrOfType<BoolAttr>(varargsAttrName);
@@ -455,7 +451,7 @@ mlir::convertFuncOpToLLVMFuncOp(FunctionOpInterface funcOp,
   // pointee type in the function body when converting `llvm.byval`/`llvm.byref`
   // function arguments.
   restoreByValRefArgumentType(rewriter, converter, byValRefNonPtrAttrs,
-                              oldBlockArgs, newFuncOp);
+                              newFuncOp);
 
   if (!shouldUseBarePtrCallConv(funcOp, &converter)) {
     if (funcOp->getAttrOfType<UnitAttr>(
-- 
cgit v1.2.3


From a5fa5bd2a890c588b99b07fc0f3fcef236888609 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 11:07:20 +0100
Subject: [X86] ptest.ll - add test coverage for #144861 load chains

---
 llvm/test/CodeGen/X86/ptest.ll | 168 ++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 164 insertions(+), 4 deletions(-)

diff --git a/llvm/test/CodeGen/X86/ptest.ll b/llvm/test/CodeGen/X86/ptest.ll
index d3da7524eaf1..6e43b897caef 100644
--- a/llvm/test/CodeGen/X86/ptest.ll
+++ b/llvm/test/CodeGen/X86/ptest.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2,-avx | FileCheck %s --check-prefix=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1,-avx | FileCheck %s --check-prefix=SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,-avx2   | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2,-avx | FileCheck %s --check-prefixes=CHECK,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1,-avx | FileCheck %s --check-prefixes=CHECK,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx,-avx2   | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=CHECK,AVX,AVX512
 
 define i32 @veccond128(<4 x i32> %input) {
 ; SSE2-LABEL: veccond128:
@@ -388,3 +388,163 @@ define i32 @vecsel512(<16 x i32> %input, i32 %a, i32 %b) {
   %t2 = select i1 %t1, i32 %a, i32 %b
   ret i32 %t2
 }
+
+define i1 @vecmp_load64x2(ptr %p0) {
+; CHECK-LABEL: vecmp_load64x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    orq 8(%rdi), %rax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 8
+  %i0 = load i64, ptr %p0, align 1
+  %i1 = load i64, ptr %p1, align 1
+  %or = or i64 %i0, %i1
+  %ne = icmp ne i64 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
+
+define i1 @vecmp_load64x4(ptr %p0) {
+; CHECK-LABEL: vecmp_load64x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movq 8(%rdi), %rcx
+; CHECK-NEXT:    orq 16(%rdi), %rax
+; CHECK-NEXT:    orq 24(%rdi), %rcx
+; CHECK-NEXT:    orq %rax, %rcx
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 8
+  %p2 = getelementptr i8, ptr %p0, i64 16
+  %p3 = getelementptr i8, ptr %p0, i64 24
+  %i0 = load i64, ptr %p0, align 1
+  %i1 = load i64, ptr %p1, align 1
+  %i2 = load i64, ptr %p2, align 1
+  %i3 = load i64, ptr %p3, align 1
+  %or02 = or i64 %i0, %i2
+  %or13 = or i64 %i1, %i3
+  %or = or i64 %or02, %or13
+  %ne = icmp ne i64 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
+
+define i1 @vecmp_load128x2(ptr %p0) {
+; CHECK-LABEL: vecmp_load128x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movq 8(%rdi), %rcx
+; CHECK-NEXT:    orq 24(%rdi), %rcx
+; CHECK-NEXT:    orq 16(%rdi), %rax
+; CHECK-NEXT:    orq %rcx, %rax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 16
+  %i0 = load i128, ptr %p0, align 1
+  %i1 = load i128, ptr %p1, align 1
+  %or = or i128 %i0, %i1
+  %ne = icmp ne i128 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
+
+define i1 @vecmp_load128x4(ptr %p0) {
+; CHECK-LABEL: vecmp_load128x4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq (%rdi), %rax
+; CHECK-NEXT:    movq 8(%rdi), %rcx
+; CHECK-NEXT:    movq 24(%rdi), %rdx
+; CHECK-NEXT:    movq 16(%rdi), %rsi
+; CHECK-NEXT:    orq 32(%rdi), %rax
+; CHECK-NEXT:    orq 40(%rdi), %rcx
+; CHECK-NEXT:    orq 48(%rdi), %rsi
+; CHECK-NEXT:    orq %rax, %rsi
+; CHECK-NEXT:    orq 56(%rdi), %rdx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    orq %rsi, %rdx
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 16
+  %p2 = getelementptr i8, ptr %p0, i64 32
+  %p3 = getelementptr i8, ptr %p0, i64 48
+  %i0 = load i128, ptr %p0, align 1
+  %i1 = load i128, ptr %p1, align 1
+  %i2 = load i128, ptr %p2, align 1
+  %i3 = load i128, ptr %p3, align 1
+  %or02 = or i128 %i0, %i2
+  %or13 = or i128 %i1, %i3
+  %or = or i128 %or02, %or13
+  %ne = icmp ne i128 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
+
+; PR144861
+define i1 @vecmp_load256x2(ptr %p0) {
+; CHECK-LABEL: vecmp_load256x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq 24(%rdi), %rax
+; CHECK-NEXT:    movq (%rdi), %rcx
+; CHECK-NEXT:    movq 8(%rdi), %rdx
+; CHECK-NEXT:    movq 16(%rdi), %rsi
+; CHECK-NEXT:    orq 48(%rdi), %rsi
+; CHECK-NEXT:    orq 32(%rdi), %rcx
+; CHECK-NEXT:    orq %rsi, %rcx
+; CHECK-NEXT:    orq 56(%rdi), %rax
+; CHECK-NEXT:    orq 40(%rdi), %rdx
+; CHECK-NEXT:    orq %rax, %rdx
+; CHECK-NEXT:    orq %rcx, %rdx
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 32
+  %i0 = load i256, ptr %p0, align 1
+  %i1 = load i256, ptr %p1, align 1
+  %or = or i256 %i0, %i1
+  %ne = icmp ne i256 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
+
+define i1 @vecmp_load512x2(ptr %p0) {
+; CHECK-LABEL: vecmp_load512x2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movq 24(%rdi), %rax
+; CHECK-NEXT:    movq 56(%rdi), %rdx
+; CHECK-NEXT:    movq 40(%rdi), %rsi
+; CHECK-NEXT:    movq 16(%rdi), %rcx
+; CHECK-NEXT:    movq 48(%rdi), %r8
+; CHECK-NEXT:    movq (%rdi), %r9
+; CHECK-NEXT:    movq 8(%rdi), %r10
+; CHECK-NEXT:    movq 32(%rdi), %r11
+; CHECK-NEXT:    orq 96(%rdi), %r11
+; CHECK-NEXT:    orq 64(%rdi), %r9
+; CHECK-NEXT:    orq %r11, %r9
+; CHECK-NEXT:    orq 112(%rdi), %r8
+; CHECK-NEXT:    orq 80(%rdi), %rcx
+; CHECK-NEXT:    orq %r8, %rcx
+; CHECK-NEXT:    orq %r9, %rcx
+; CHECK-NEXT:    orq 104(%rdi), %rsi
+; CHECK-NEXT:    orq 72(%rdi), %r10
+; CHECK-NEXT:    orq %rsi, %r10
+; CHECK-NEXT:    orq 120(%rdi), %rdx
+; CHECK-NEXT:    orq 88(%rdi), %rax
+; CHECK-NEXT:    orq %rdx, %rax
+; CHECK-NEXT:    orq %r10, %rax
+; CHECK-NEXT:    orq %rcx, %rax
+; CHECK-NEXT:    sete %al
+; CHECK-NEXT:    retq
+  %p1 = getelementptr i8, ptr %p0, i64 64
+  %i0 = load i512, ptr %p0, align 1
+  %i1 = load i512, ptr %p1, align 1
+  %or = or i512 %i0, %i1
+  %ne = icmp ne i512 %or, 0
+  %zx = zext i1 %ne to i32
+  %eq = icmp eq i32 %zx, 0
+  ret i1 %eq
+}
-- 
cgit v1.2.3


From 7085065c02da6091dca91be201160912e43a63ec Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 20 Jun 2025 03:17:08 -0700
Subject: [BOLT] Support pre-aggregated returns (#143296)

Intel's Architectural LBR supports capturing branch type information
as part of LBR stack (SDM Vol 3B, part 2, October 2024):
```
20.1.3.2 Branch Types
The IA32_LBR_x_INFO.BR_TYPE and IA32_LER_INFO.BR_TYPE fields encode
the branch types as shown in Table 20-3.

Table 20-3. IA32_LBR_x_INFO and IA32_LER_INFO Branch Type Encodings

Encoding | Branch Type
  0000B  | COND
  0001B  | NEAR_IND_JMP
  0010B  | NEAR_REL_JMP
  0011B  | NEAR_IND_CALL
  0100B  | NEAR_REL_CALL
  0101B  | NEAR_RET
  011xB  | Reserved
  1xxxB  | OTHER_BRANCH

For a list of branch operations that fall into the categories above,
see Table 20-2.

Table 20-2. Branch Type Filtering Details
Branch Type   | Operations Recorded
COND          | Jcc, J*CXZ, and LOOP*
NEAR_IND_JMP  | JMP r/m*
NEAR_REL_JMP  | JMP rel*
NEAR_IND_CALL | CALL r/m*
NEAR_REL_CALL | CALL rel* (excluding CALLs to the next sequential IP)
NEAR_RET      | RET (0C3H)
OTHER_BRANCH  | JMP/CALL ptr*, JMP/CALL m*, RET (0C8H), SYS*,
interrupts, exceptions (other than debug exceptions), IRET, INT3,
INTn, INTO, TSX Abort, EENTER, ERESUME, EEXIT, AEX, INIT, SIPI, RSM
```

Linux kernel can preserve branch type when `save_type` is enabled,
even if CPU does not support Architectural LBR:

https://github.com/torvalds/linux/blob/f09079bd04a924c72d555cd97942d5f8d7eca98c/tools/perf/Documentation/perf-record.txt#L457-L460

> - save_type: save branch type during sampling in case binary is not
available later.
For the platforms with Intel Arch LBR support (12th-Gen+ client or
4th-Gen Xeon+ server), the save branch type is unconditionally enabled
when the taken branch stack sampling is enabled.

Kernel-reported branch type values:

https://github.com/torvalds/linux/blob/8c6bc74c7f8910ed4c969ccec52e98716f98700a/include/uapi/linux/perf_event.h#L251-L269

This information is needed to disambiguate external returns (from
DSO/JIT) to an entry point or a landing pad, when BOLT can't
disassemble the branch source.

This patch adds new pre-aggregated types:
- return trace (R),
- external return fall-through (r).

For such types, the checks for fall-through start (not an entry or
a landing pad) are relaxed.

Depends on #143295.

Test Plan: updated callcont-fallthru.s
---
 bolt/include/bolt/Profile/DataAggregator.h | 23 ++++++++----
 bolt/lib/Profile/DataAggregator.cpp        | 59 ++++++++++++++++++------------
 bolt/test/X86/callcont-fallthru.s          | 23 ++++++++++++
 bolt/test/link_fdata.py                    |  4 +-
 4 files changed, 77 insertions(+), 32 deletions(-)

diff --git a/bolt/include/bolt/Profile/DataAggregator.h b/bolt/include/bolt/Profile/DataAggregator.h
index 662fe2a49afe..cc28a06c151e 100644
--- a/bolt/include/bolt/Profile/DataAggregator.h
+++ b/bolt/include/bolt/Profile/DataAggregator.h
@@ -101,16 +101,17 @@ private:
     uint64_t Addr;
   };
 
-  /// Container for the unit of branch data.
-  /// Backwards compatible with legacy use for branches and fall-throughs:
-  /// - if \p Branch is FT_ONLY or FT_EXTERNAL_ORIGIN, the trace only
-  ///   contains fall-through data,
-  /// - if \p To is BR_ONLY, the trace only contains branch data.
+  /// Container for the unit of branch data, matching pre-aggregated trace type.
+  /// Backwards compatible with branch and fall-through types:
+  /// - if \p To is < 0, the trace only contains branch data (BR_ONLY),
+  /// - if \p Branch is < 0, the trace only contains fall-through data
+  ///   (FT_ONLY, FT_EXTERNAL_ORIGIN, or FT_EXTERNAL_RETURN).
   struct Trace {
     static constexpr const uint64_t EXTERNAL = 0ULL;
     static constexpr const uint64_t BR_ONLY = -1ULL;
     static constexpr const uint64_t FT_ONLY = -1ULL;
     static constexpr const uint64_t FT_EXTERNAL_ORIGIN = -2ULL;
+    static constexpr const uint64_t FT_EXTERNAL_RETURN = -3ULL;
 
     uint64_t Branch;
     uint64_t From;
@@ -390,9 +391,9 @@ private:
   /// File format syntax:
   /// E <event>
   /// S <start> <count>
-  /// T <start> <end> <ft_end> <count>
+  /// [TR] <start> <end> <ft_end> <count>
   /// B <start> <end> <count> <mispred_count>
-  /// [Ff] <start> <end> <count>
+  /// [Ffr] <start> <end> <count>
   ///
   /// where <start>, <end>, <ft_end> have the format [<id>:]<offset>
   ///
@@ -403,8 +404,11 @@ private:
   /// f - an aggregated fall-through with external origin - used to disambiguate
   ///       between a return hitting a basic block head and a regular internal
   ///       jump to the block
+  /// r - an aggregated fall-through originating at an external return, no
+  ///       checks are performed for a fallthrough start
   /// T - an aggregated trace: branch from <start> to <end> with a fall-through
   ///       to <ft_end>
+  /// R - an aggregated trace originating at a return
   ///
   /// <id> - build id of the object containing the address. We can skip it for
   /// the main binary and use "X" for an unknown object. This will save some
@@ -532,7 +536,12 @@ inline raw_ostream &operator<<(raw_ostream &OS,
                                const DataAggregator::Trace &T) {
   switch (T.Branch) {
   case DataAggregator::Trace::FT_ONLY:
+    break;
   case DataAggregator::Trace::FT_EXTERNAL_ORIGIN:
+    OS << "X:0 -> ";
+    break;
+  case DataAggregator::Trace::FT_EXTERNAL_RETURN:
+    OS << "X:R -> ";
     break;
   default:
     OS << Twine::utohexstr(T.Branch) << " -> ";
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index e03fa9bd5322..c067b2f5b73b 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -537,8 +537,7 @@ Error DataAggregator::preprocessProfile(BinaryContext &BC) {
 
 heatmap:
   // Sort parsed traces for faster processing.
-  if (!opts::BasicAggregation)
-    llvm::sort(Traces, llvm::less_first());
+  llvm::sort(Traces, llvm::less_first());
 
   if (!opts::HeatmapMode)
     return Error::success();
@@ -883,13 +882,9 @@ DataAggregator::getFallthroughsInTrace(BinaryFunction &BF, const Trace &Trace,
 
   // Adjust FromBB if the first LBR is a return from the last instruction in
   // the previous block (that instruction should be a call).
-  if (IsReturn) {
-    if (From)
-      FromBB = BF.getBasicBlockContainingOffset(From - 1);
-    else
-      LLVM_DEBUG(dbgs() << "return to the function start: " << Trace << '\n');
-  } else if (Trace.Branch == Trace::EXTERNAL && From == FromBB->getOffset() &&
-             !FromBB->isEntryPoint() && !FromBB->isLandingPad()) {
+  if (Trace.Branch != Trace::FT_ONLY && !BF.containsAddress(Trace.Branch) &&
+      From == FromBB->getOffset() &&
+      (IsReturn ? From : !(FromBB->isEntryPoint() || FromBB->isLandingPad()))) {
     const BinaryBasicBlock *PrevBB =
         BF.getLayout().getBlock(FromBB->getIndex() - 1);
     if (PrevBB->getSuccessor(FromBB->getLabel())) {
@@ -1228,12 +1223,14 @@ ErrorOr<Location> DataAggregator::parseLocationOrOffset() {
 std::error_code DataAggregator::parseAggregatedLBREntry() {
   enum AggregatedLBREntry : char {
     INVALID = 0,
-    EVENT_NAME,        // E
-    TRACE,             // T
-    SAMPLE,            // S
-    BRANCH,            // B
-    FT,                // F
-    FT_EXTERNAL_ORIGIN // f
+    EVENT_NAME,         // E
+    TRACE,              // T
+    RETURN,             // R
+    SAMPLE,             // S
+    BRANCH,             // B
+    FT,                 // F
+    FT_EXTERNAL_ORIGIN, // f
+    FT_EXTERNAL_RETURN  // r
   } Type = INVALID;
 
   /// The number of fields to parse, set based on \p Type.
@@ -1261,20 +1258,22 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
 
     Type = StringSwitch<AggregatedLBREntry>(Str)
                .Case("T", TRACE)
+               .Case("R", RETURN)
                .Case("S", SAMPLE)
                .Case("E", EVENT_NAME)
                .Case("B", BRANCH)
                .Case("F", FT)
                .Case("f", FT_EXTERNAL_ORIGIN)
+               .Case("r", FT_EXTERNAL_RETURN)
                .Default(INVALID);
 
     if (Type == INVALID) {
-      reportError("expected T, S, E, B, F or f");
+      reportError("expected T, R, S, E, B, F, f or r");
       return make_error_code(llvm::errc::io_error);
     }
 
     using SSI = StringSwitch<int>;
-    AddrNum = SSI(Str).Case("T", 3).Case("S", 1).Case("E", 0).Default(2);
+    AddrNum = SSI(Str).Cases("T", "R", 3).Case("S", 1).Case("E", 0).Default(2);
     CounterNum = SSI(Str).Case("B", 2).Case("E", 0).Default(1);
   }
 
@@ -1331,17 +1330,30 @@ std::error_code DataAggregator::parseAggregatedLBREntry() {
   if (ToFunc)
     ToFunc->setHasProfileAvailable();
 
-  /// For legacy fall-through types, adjust locations to match Trace container.
-  if (Type == FT || Type == FT_EXTERNAL_ORIGIN) {
+  /// For fall-through types, adjust locations to match Trace container.
+  if (Type == FT || Type == FT_EXTERNAL_ORIGIN || Type == FT_EXTERNAL_RETURN) {
     Addr[2] = Location(Addr[1]->Offset); // Trace To
     Addr[1] = Location(Addr[0]->Offset); // Trace From
-    // Put a magic value into Trace Branch to differentiate from a full trace.
-    Addr[0] = Location(Type == FT ? Trace::FT_ONLY : Trace::FT_EXTERNAL_ORIGIN);
+    // Put a magic value into Trace Branch to differentiate from a full trace:
+    if (Type == FT)
+      Addr[0] = Location(Trace::FT_ONLY);
+    else if (Type == FT_EXTERNAL_ORIGIN)
+      Addr[0] = Location(Trace::FT_EXTERNAL_ORIGIN);
+    else if (Type == FT_EXTERNAL_RETURN)
+      Addr[0] = Location(Trace::FT_EXTERNAL_RETURN);
+    else
+      llvm_unreachable("Unexpected fall-through type");
   }
 
-  /// For legacy branch type, mark Trace To to differentite from a full trace.
-  if (Type == BRANCH) {
+  /// For branch type, mark Trace To to differentiate from a full trace.
+  if (Type == BRANCH)
     Addr[2] = Location(Trace::BR_ONLY);
+
+  if (Type == RETURN) {
+    if (!Addr[0]->Offset)
+      Addr[0]->Offset = Trace::FT_EXTERNAL_RETURN;
+    else
+      Returns.emplace(Addr[0]->Offset);
   }
 
   /// Record a trace.
@@ -1602,6 +1614,7 @@ void DataAggregator::processBranchEvents() {
   NamedRegionTimer T("processBranch", "Processing branch events",
                      TimerGroupName, TimerGroupDesc, opts::TimeAggregator);
 
+  Returns.emplace(Trace::FT_EXTERNAL_RETURN);
   for (const auto &[Trace, Info] : Traces) {
     bool IsReturn = checkReturn(Trace.Branch);
     // Ignore returns.
diff --git a/bolt/test/X86/callcont-fallthru.s b/bolt/test/X86/callcont-fallthru.s
index c2ef024db947..8c05491e7bca 100644
--- a/bolt/test/X86/callcont-fallthru.s
+++ b/bolt/test/X86/callcont-fallthru.s
@@ -10,6 +10,10 @@
 # RUN: link_fdata %s %t %t.pa-ret PREAGG-RET
 # Trace from an external location to a landing pad/entry point call continuation
 # RUN: link_fdata %s %t %t.pa-ext PREAGG-EXT
+# Return trace to a landing pad/entry point call continuation
+# RUN: link_fdata %s %t %t.pa-pret PREAGG-PRET
+# External return to a landing pad/entry point call continuation
+# RUN: link_fdata %s %t %t.pa-eret PREAGG-ERET
 # RUN-DISABLED: link_fdata %s %t %t.pa-plt PREAGG-PLT
 
 # RUN: llvm-strip --strip-unneeded %t -o %t.strip
@@ -38,6 +42,21 @@
 # RUN: llvm-bolt %t.strip --pa -p %t.pa-ext -o %t.out \
 # RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK-SKIP
 
+## Check pre-aggregated return traces from external location attach call
+## continuation fallthrough count to secondary entry point (unstripped)
+# RUN: llvm-bolt %t --pa -p %t.pa-pret -o %t.out \
+# RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK-ATTACH
+## Check pre-aggregated return traces from external location attach call
+## continuation fallthrough count to landing pad (stripped, landing pad)
+# RUN: llvm-bolt %t.strip --pa -p %t.pa-pret -o %t.out \
+# RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK-ATTACH
+
+## Same for external return type
+# RUN: llvm-bolt %t --pa -p %t.pa-eret -o %t.out \
+# RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK-ATTACH
+# RUN: llvm-bolt %t.strip --pa -p %t.pa-eret -o %t.out \
+# RUN:   --print-cfg --print-only=main | FileCheck %s --check-prefix=CHECK-ATTACH
+
 ## Check pre-aggregated traces don't report zero-sized PLT fall-through as
 ## invalid trace
 # RUN-DISABLED: llvm-bolt %t.strip --pa -p %t.pa-plt -o %t.out | FileCheck %s \
@@ -92,6 +111,10 @@ Ltmp4_br:
 # PREAGG-RET: T #Lfoo_ret# #Ltmp3# #Ltmp3_br# 1
 ## Target is a secondary entry point (unstripped) or a landing pad (stripped)
 # PREAGG-EXT: T X:0 #Ltmp3# #Ltmp3_br# 1
+## Pre-aggregated return trace
+# PREAGG-PRET: R X:0 #Ltmp3# #Ltmp3_br# 1
+## External return
+# PREAGG-ERET: r #Ltmp3# #Ltmp3_br# 1
 
 # CHECK-ATTACH:      callq foo
 # CHECK-ATTACH-NEXT: count: 1
diff --git a/bolt/test/link_fdata.py b/bolt/test/link_fdata.py
index 5a9752068bb9..898dce8e3fb5 100755
--- a/bolt/test/link_fdata.py
+++ b/bolt/test/link_fdata.py
@@ -36,9 +36,9 @@ prefix_pat = re.compile(f"^# {args.prefix}: (.*)")
 fdata_pat = re.compile(r"([01].*) (?P<mispred>\d+) (?P<exec>\d+)")
 
 # Pre-aggregated profile:
-# {T|S|E|B|F|f} <start> [<end>] [<ft_end>] <count> [<mispred_count>]
+# {T|R|S|E|B|F|f|r} <start> [<end>] [<ft_end>] <count> [<mispred_count>]
 # <loc>: [<id>:]<offset>
-preagg_pat = re.compile(r"(?P<type>[TSBFf]) (?P<offsets_count>.*)")
+preagg_pat = re.compile(r"(?P<type>[TRSBFfr]) (?P<offsets_count>.*)")
 
 # No-LBR profile:
 # <is symbol?> <closest elf symbol or DSO name> <relative address> <count>
-- 
cgit v1.2.3


From 00c18d04ab6341022867d3b6674ec3ab30e5de2c Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <Nico.Vasilache@amd.com>
Date: Fri, 20 Jun 2025 12:31:46 +0200
Subject: =?UTF-8?q?[mlir][Transforms]=20Add=20a=20PadTilingInterface=20tra?=
 =?UTF-8?q?nsformation=20and=20hook=20i=E2=80=A6=20(#144991)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…t up to the transform dialect

This revision revisits the padding transformation from first principles
and prepares it to work more generally with TilingInterface.

Compared to structured.transform.pad it has the following differences:
- no support for nofold, copy-back, transpose and hoisting: these have
been carried by the padding op in the very early days of StructuredOps
and have since then been separated out as independent transformations
that compose.
- no conflated static bounding box derivation attempts:
pad_tiling_interface composes more naturally with or without tiling.
- properly derives padding size on outputs where multiple dimensions
contribute: this is not supported in structured.transform.pad
- geared towards supporting TilingInterface once the proper control
mechanisms are supported through a PadSizeComputationFunction (supports
LinalgOp by default)

This will gradually replace structured.transform.pad as it is fleshed
out and tested more comprehensively.

In the future this should be moved out of a specific Linalg
implementation file and into a more general "Structured" file.
---
 .../Linalg/TransformOps/LinalgTransformOps.td      |  79 +++++
 .../mlir/Dialect/Linalg/Transforms/Transforms.h    |  79 ++++-
 .../Linalg/TransformOps/LinalgTransformOps.cpp     | 161 +++++++++++
 mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt  |   1 +
 .../Linalg/Transforms/PadTilingInterface.cpp       | 322 +++++++++++++++++++++
 ...nsform-op-pad-tiling-interface-multiple-of.mlir | 138 +++++++++
 .../Linalg/transform-op-pad-tiling-interface.mlir  | 129 +++++++++
 7 files changed, 906 insertions(+), 3 deletions(-)
 create mode 100644 mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
 create mode 100644 mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
 create mode 100644 mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir

diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 6f6df350f1ba..cf3f2b70580d 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -1186,6 +1186,85 @@ def PadOp : Op<Transform_Dialect, "structured.pad",
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// PadTilingInterfaceOp
+//===----------------------------------------------------------------------===//
+
+def PadTilingInterfaceOp : Op<Transform_Dialect, "structured.pad_tiling_interface",
+    [FunctionalStyleTransformOpTrait, DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+     TransformOpInterface,
+     ReportTrackingListenerFailuresOpTrait]> {
+  let description = [{
+    Pads the operations pointed to by the target handle using the options
+    provided as operation attributes. The operation returns a handle to the
+    padded operation and to the padding operation ("tensor.pad").
+
+    TODO: in the future this should be moved out of a specific Linalg
+    implementation file and into a more general "Structured" file.
+
+    #### Return modes
+
+    This operation ignores non-Linalg ops and drops them in the return.
+    In the future, this operation will support all TilingInterfaceOps.
+
+    This operation may produce a definite failure if the padding fails for any
+    reason.
+
+    If all the operations referred to by the `target` handle pad properly, the
+    transform succeeds. Otherwise the transform produces a silenceable failure.
+    The return handle points to only the subset of successfully produced
+    padded operations, which can be empty.
+  }];
+
+  let arguments =
+    (ins TransformHandleTypeInterface:$target,
+         DefaultValuedAttr<ArrayAttr, "{}">:$padding_values,
+         DefaultValuedAttr<I64ArrayAttr, "{}">:$padding_dimensions,
+         Variadic<TransformAnyParamTypeOrAnyHandle>:$padding_sizes,
+         DefaultValuedOptionalAttr<DenseI64ArrayAttr, "{}">:
+            $static_padding_sizes,
+         DefaultValuedAttr<UnitAttr, "false">:$pad_to_multiple_of);
+  let results = (outs TransformHandleTypeInterface:$padded,
+                      TransformHandleTypeInterface:$pad);
+
+  let assemblyFormat = [{
+    $target
+    `to`
+    (`padding_sizes` custom<DynamicIndexList>($padding_sizes, $static_padding_sizes)^)?
+    (`pad_to_multiple_of` $pad_to_multiple_of^)?
+    attr-dict
+    `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+
+  let builders = [
+    // Builder for a transform::PadOp with automatic inference of padding
+    // value. Warning: this will set the value 0 for the inferred elemental
+    // type without taking the op into account and thus only work for the
+    // add/mul ring at the moment.
+    // TODO: support other operations (e.g. min, max etc).
+    OpBuilder<(ins "Value":$target,
+                   "ArrayRef<int64_t>":$paddingDimensions,
+                   CArg<"ArrayRef<int64_t>", "{}">:$staticPaddingSizes,
+                   CArg<"bool", "false">:$padToMultipleOf)>,
+    OpBuilder<(ins "Value":$target,
+                   "ArrayRef<int64_t>":$paddingDimensions,
+                   "ArrayRef<OpFoldResult>":$mixedPadPaddingSizes,
+                   CArg<"bool", "false">:$usePrescribedTensorShapes)>
+  ];
+
+  let extraClassDeclaration = [{
+    /// Returns a mix of dynamic `padding_sizes` and static `static_padding_sizes`.
+    SmallVector<OpFoldResult> getMixedPaddingSizes();
+
+    ::mlir::DiagnosedSilenceableFailure apply(
+      ::mlir::transform::TransformRewriter &rewriter,
+      ::mlir::transform::TransformResults &results,
+      ::mlir::transform::TransformState &state);
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // HoistPadOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 147a2907f52e..59b7fdeef10b 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -20,6 +20,7 @@
 #include "mlir/Dialect/Utils/StaticValueUtils.h"
 #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
 #include "mlir/Dialect/X86Vector/Transforms.h"
+#include "mlir/IR/OpDefinition.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/TilingInterface.h"
 #include "mlir/Transforms/DialectConversion.h"
@@ -347,6 +348,34 @@ struct LinalgPaddingOptions {
   }
 };
 
+struct PadTilingInterfaceOptions {
+  /// A padding value for every operand.
+  SmallVector<Attribute> paddingValues;
+  PadTilingInterfaceOptions &setPaddingValues(ArrayRef<Attribute> pv) {
+    paddingValues.assign(pv.begin(), pv.end());
+    return *this;
+  }
+  /// A list of iterator dimensions to pad.
+  SmallVector<int64_t> paddingDimensions;
+  PadTilingInterfaceOptions &setPaddingDimensions(ArrayRef<int64_t> pd) {
+    paddingDimensions.assign(pd.begin(), pd.end());
+    return *this;
+  }
+  /// A list of iterator dimensions sizes to pad to.
+  SmallVector<OpFoldResult> paddingSizes;
+  PadTilingInterfaceOptions &setPaddingSizes(ArrayRef<OpFoldResult> m) {
+    paddingSizes.assign(m.begin(), m.end());
+    return *this;
+  }
+  /// Pad iterator `paddingDimension[i]` to next multiple of `paddingSizes[i]`
+  /// if true. Otherwise pad to `paddingSizes[i]`.
+  bool padToMultipleOf;
+  PadTilingInterfaceOptions &setPadToMultipleOf(bool b) {
+    padToMultipleOf = b;
+    return *this;
+  }
+};
+
 /// Callback function type used to perform the allocation for the promoted
 /// `subView`. In `boundingSubViewsize` a best attempt is made to find the
 /// smallest constant value for the size of the buffer needed for each
@@ -542,9 +571,9 @@ SmallVector<Value> peelLoop(RewriterBase &rewriter, Operation *op);
 /// where relevant.
 void peelLoops(RewriterBase &rewriter, ArrayRef<scf::ForOp> loops);
 
-/// Pad the iterator dimensions `paddingDimensions` of all `opToPad` operands
-/// to a static bounding box. The original `opToPad` is cloned and operates on
-/// the padded tensors.
+/// Pad the iterator dimensions `options.paddingDimensions` of all `opToPad`
+/// operands to a static bounding box. The original `opToPad` is cloned and
+/// operates on the padded tensors.
 ///
 /// * "options.padToMultipleOf" indicates that each padding dimension should be
 ///   padded to the specified multiple.
@@ -561,6 +590,50 @@ LogicalResult rewriteAsPaddedOp(RewriterBase &rewriter, LinalgOp opToPad,
                                 SmallVector<Value> &replacements,
                                 SmallVector<tensor::PadOp> &padOps);
 
+/// Helper function to compute the padded shape of the given value `v` of
+/// `RankedTensorType` given:
+///   - the `indexingSizes` as a list of OpFoldResult.
+///   - an `indexingMap` that encodes how the padded shape varies with
+///     increases in `indexingSizes`.
+/// The implementation iteratively combines increases from contributing using
+/// affine.apply operations.
+/// The `indexingMap` + `indexingSizes` encoding suits StructuredOps and
+/// provides a gentle portability path for Linalg-like ops with affine maps.
+/// In the future, more general interfaces can be devised to encode similar
+/// shape evolutions and map between an op and its operands.
+SmallVector<OpFoldResult>
+computePaddedShape(RewriterBase &rewriter, TypedValue<RankedTensorType> v,
+                   AffineMap indexingMap, ArrayRef<OpFoldResult> indexingSizes,
+                   const PadTilingInterfaceOptions &options);
+
+using PadSizeComputationFunction =
+    std::function<FailureOr<SmallVector<OpFoldResult>>(
+        RewriterBase &, OpOperand &, ArrayRef<Range>,
+        const PadTilingInterfaceOptions &)>;
+
+/// Specific helper for Linalg ops.
+FailureOr<SmallVector<OpFoldResult>>
+computeLinalgPaddedShape(RewriterBase &rewriter, OpOperand &operandToPad,
+                         ArrayRef<Range> iterationDomain,
+                         const PadTilingInterfaceOptions &options);
+
+/// Pad the iterator dimensions `options.paddingDimensions` of `opToPad`.
+///
+/// * "options.paddingSizes" indicates that each padding dimension should be
+///   padded to the specified padding size.
+/// * "options.padToMultipleOf" indicates that the paddingSizes should be
+//    interpreted as the bounding box (dynamic) value to pad to.
+/// * Use "options.paddingValues" to set the padding value of the created
+//    tensor::PadOp.
+/// * The tensor::PadOp is returned on success.
+
+FailureOr<TilingInterface>
+rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
+                  const PadTilingInterfaceOptions &constOptions,
+                  SmallVector<tensor::PadOp> &padOps,
+                  PadSizeComputationFunction computePaddingSizeFun =
+                      &computeLinalgPaddedShape);
+
 namespace detail {
 
 /// Helper struct to hold the results of building a packing loop nest.
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index d78c8847f884..e627fc83f2ba 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -45,6 +45,7 @@
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/LogicalResult.h"
 #include <type_traits>
 
 using namespace mlir;
@@ -2155,6 +2156,166 @@ LogicalResult transform::PadOp::verify() {
   return success();
 }
 
+//===---------------------------------------------------------------------===//
+// PadTilingInterfaceOp
+//===---------------------------------------------------------------------===//
+
+void transform::PadTilingInterfaceOp::build(OpBuilder &b,
+                                            OperationState &result,
+                                            Value target,
+                                            ArrayRef<int64_t> paddingDimensions,
+                                            ArrayRef<int64_t> paddingSizes,
+                                            bool padToMultipleOf) {
+  auto resultType = transform::AnyOpType::get(b.getContext());
+  return build(/*builder=*/b,
+               /*result=*/result,
+               /*types=*/TypeRange{resultType, resultType},
+               /*target=*/target,
+               /*paddingValues=*/ArrayAttr(), // let inference handle this
+               /*paddingDimensions=*/b.getI64ArrayAttr(paddingDimensions),
+               /*paddingSizes=*/ValueRange{},
+               /*paddingSizes=*/
+               (paddingSizes.empty() ? DenseI64ArrayAttr()
+                                     : b.getDenseI64ArrayAttr(paddingSizes)),
+               /*padToMultipleOf=*/
+               padToMultipleOf ? b.getUnitAttr() : nullptr);
+}
+
+void transform::PadTilingInterfaceOp::build(
+    OpBuilder &b, OperationState &result, Value target,
+    ArrayRef<int64_t> paddingDimensions,
+    ArrayRef<OpFoldResult> mixedPaddingSizes, bool padToMultipleOf) {
+  auto resultType = transform::AnyOpType::get(b.getContext());
+  SmallVector<int64_t> staticPaddingSizes;
+  SmallVector<Value> dynamicPaddingSizes;
+  dispatchIndexOpFoldResults(mixedPaddingSizes, dynamicPaddingSizes,
+                             staticPaddingSizes);
+  return build(/*builder=*/b,
+               /*result=*/result,
+               /*types=*/TypeRange{resultType, resultType},
+               /*target=*/target,
+               /*paddingValues=*/ArrayAttr(), // let inference handle this
+               /*paddingDimensions=*/b.getI64ArrayAttr(paddingDimensions),
+               /*paddingSizes=*/dynamicPaddingSizes,
+               /*paddingSizes=*/staticPaddingSizes,
+               /*usePrescribedTensorShapes=*/padToMultipleOf);
+}
+
+void transform::PadTilingInterfaceOp::getEffects(
+    SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {
+  consumesHandle(getTargetMutable(), effects);
+  onlyReadsHandle(getPaddingSizesMutable(), effects);
+  producesHandle(getOperation()->getOpResults(), effects);
+  modifiesPayload(effects);
+}
+
+SmallVector<OpFoldResult>
+transform::PadTilingInterfaceOp::getMixedPaddingSizes() {
+  Builder b(getContext());
+  return getMixedValues(getStaticPaddingSizes(), getPaddingSizes(), b);
+}
+
+DiagnosedSilenceableFailure
+transform::PadTilingInterfaceOp::apply(transform::TransformRewriter &rewriter,
+                                       transform::TransformResults &results,
+                                       transform::TransformState &state) {
+  SmallVector<Operation *> paddedOps, padOps;
+
+  for (Operation *target : state.getPayloadOps(getTarget())) {
+    auto targetOp = dyn_cast<TilingInterface>(target);
+    if (!targetOp) {
+      auto diag = emitSilenceableError() << "expected TilingInterface target";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
+
+    // Only Linalg ops for now, until TilingInterface exposes a loopsToOperand
+    // map / C++ APIs to compute the effect of padding on operands.
+    if (!isa<LinalgOp>(targetOp.getOperation())) {
+      auto diag = emitSilenceableError() << "only LinalgOp supported atm";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
+
+    // Convert the padding values to attributes.
+    SmallVector<Attribute> paddingValues;
+    for (auto const &[untypedAttr, elementOrTensorType] :
+         llvm::zip(getPaddingValues(), targetOp->getOperandTypes())) {
+      auto attr = dyn_cast<TypedAttr>(untypedAttr);
+      Type elementType = getElementTypeOrSelf(elementOrTensorType);
+      if (!attr) {
+        emitOpError("expects padding values to be typed attributes");
+        return DiagnosedSilenceableFailure::definiteFailure();
+      }
+      // Try to parse string attributes to obtain an attribute of element type.
+      if (auto stringAttr = dyn_cast<StringAttr>(attr)) {
+        auto parsedAttr = dyn_cast_if_present<TypedAttr>(parseAttribute(
+            stringAttr, getContext(), elementType,
+            /*numRead=*/nullptr, /*isKnownNullTerminated=*/true));
+        if (!parsedAttr || parsedAttr.getType() != elementType) {
+          auto diag = this->emitOpError("expects a padding that parses to ")
+                      << elementType << ", got " << attr;
+          diag.attachNote(targetOp.getLoc()) << "when applied to this op";
+          return DiagnosedSilenceableFailure::definiteFailure();
+        }
+        paddingValues.push_back(parsedAttr);
+        continue;
+      }
+      // Otherwise, add the attribute directly.
+      if (attr.getType() != elementType) {
+        auto diag = this->emitOpError("expects a padding value of type ")
+                    << elementType << ", got " << attr;
+        diag.attachNote(targetOp.getLoc()) << "when applied to this op";
+        return DiagnosedSilenceableFailure::definiteFailure();
+      }
+      paddingValues.push_back(attr);
+    }
+
+    // Set options.
+    TilingInterface paddedOp;
+    PadTilingInterfaceOptions options;
+    options.setPaddingValues(paddingValues)
+        .setPaddingDimensions(
+            extractFromIntegerArrayAttr<int64_t>(getPaddingDimensions()))
+        .setPaddingSizes(getMixedPaddingSizes())
+        .setPadToMultipleOf(getPadToMultipleOf());
+
+    // Apply padding.
+    SmallVector<tensor::PadOp> newPadOps;
+    FailureOr<TilingInterface> maybePaddedOp = rewriteAsPaddedOp(
+        rewriter, cast<TilingInterface>(targetOp.getOperation()), options,
+        newPadOps);
+    if (failed(maybePaddedOp)) {
+      auto diag = emitSilenceableError() << "failed to pad op";
+      diag.attachNote(target->getLoc()) << "target op";
+      return diag;
+    }
+
+    // Set transform results.
+    paddedOps.push_back(cast<TilingInterface>(maybePaddedOp->getOperation()));
+    padOps.append(newPadOps.begin(), newPadOps.end());
+  }
+
+  results.set(cast<OpResult>(getPadded()), paddedOps);
+  results.set(cast<OpResult>(getPad()), padOps);
+  return DiagnosedSilenceableFailure::success();
+}
+
+LogicalResult transform::PadTilingInterfaceOp::verify() {
+  SmallVector<int64_t> paddingDimensions =
+      extractFromIntegerArrayAttr<int64_t>(getPaddingDimensions());
+  if (any_of(paddingDimensions,
+             [](int64_t paddingDimension) { return paddingDimension < 0; })) {
+    return emitOpError() << "expects padding_dimensions to contain positive "
+                            "integers, found "
+                         << getPaddingDimensions();
+  }
+  if (getMixedPaddingSizes().size() != paddingDimensions.size()) {
+    return emitOpError() << "expects as many multiples as padding_dimensions";
+  }
+  return success();
+}
+
 //===---------------------------------------------------------------------===//
 // HoistPadOp
 //===---------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
index 881d9fcb4f52..69e6fdabf9a5 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/Linalg/Transforms/CMakeLists.txt
@@ -29,6 +29,7 @@ add_mlir_dialect_library(MLIRLinalgTransforms
   BlockPackMatmul.cpp
   PackAndUnpackPatterns.cpp
   Padding.cpp
+  PadTilingInterface.cpp
   Promotion.cpp
   RuntimeOpVerification.cpp
   Specialize.cpp
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
new file mode 100644
index 000000000000..a9d7bc64f2a6
--- /dev/null
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -0,0 +1,322 @@
+//===- PaddingTilingInterface.cpp - Padding of TilingInterface ops --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
+
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Complex/IR/Complex.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Utils/StaticValueUtils.h"
+#include "mlir/IR/AffineExpr.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypeInterfaces.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/OpDefinition.h"
+#include "mlir/IR/Value.h"
+#include "mlir/Interfaces/TilingInterface.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/Casting.h"
+
+#define DEBUG_TYPE "pad-tiling-interface"
+
+using namespace mlir;
+using namespace mlir::linalg;
+using namespace mlir::tensor;
+
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE << "]: ")
+#define DBGSNL() (llvm::dbgs() << "\n")
+
+/// Form a "full-rank" padding specification so that the application is easy.
+static llvm::SmallDenseMap<int64_t, OpFoldResult>
+getDimsToSize(Builder &b, ArrayRef<OpFoldResult> indexingSizes,
+              const PadTilingInterfaceOptions &options) {
+  llvm::SmallDenseMap<int64_t, OpFoldResult> dimsToSize;
+  for (const auto &[paddingDim, paddingSize] :
+       llvm::zip_equal(options.paddingDimensions, options.paddingSizes)) {
+    dimsToSize[paddingDim] = paddingSize;
+  }
+  // Complete the padding specification to specify all dimensions.
+  for (int64_t idx = 0, e = indexingSizes.size(); idx != e; ++idx) {
+    if (dimsToSize.find(idx) != dimsToSize.end())
+      continue;
+    // If a dimension is not specified, either complete with:
+    //   - 1 if we are padding to the next multiple of.
+    //   - indexingSizes[idx] otherwise
+    dimsToSize[idx] =
+        options.padToMultipleOf ? b.getIndexAttr(1) : indexingSizes[idx];
+  }
+  for (int64_t idx = 0, e = indexingSizes.size(); idx != e; ++idx) {
+    LLVM_DEBUG(DBGS() << "----idx: " << idx << " : " << dimsToSize[idx]
+                      << "\n");
+  }
+  return dimsToSize;
+}
+
+/// Compute the padded shape of the given value `v` of `RankedTensorType` given
+///   - `indexingSizes` a list of OpFoldResult.
+///   - an `indexingMap` that encodes how the shape of varies with increases
+///     in `indexingSizes`.
+/// The `indexingMap` encodes how the shape of varies with `indexingSizes`.
+/// The `indexingMap` + `indexingSizes` encoding suits StructuredOps.
+/// The implementaiton below iteratively combines increases from contributing
+/// dimensions using affine.apply operations.
+/// In the future, more general interfaces can be devised to encode similar
+/// shape evolutions and map between an op and its operands.
+SmallVector<OpFoldResult> linalg::computePaddedShape(
+    RewriterBase &rewriter, TypedValue<RankedTensorType> v,
+    AffineMap indexingMap, ArrayRef<OpFoldResult> indexingSizes,
+    const PadTilingInterfaceOptions &options) {
+  Location loc = v.getLoc();
+  SmallVector<OpFoldResult> paddedShape;
+  auto tensorType = cast<RankedTensorType>(v.getType());
+  paddedShape.resize_for_overwrite(tensorType.getRank());
+  assert(tensorType.getRank() == indexingMap.getNumResults() &&
+         "expect the number of results of the affine map to match the tensor "
+         "rank");
+
+  // "Full-rank" padding specification.
+  llvm::SmallDenseMap<int64_t, OpFoldResult> dimsToSize =
+      getDimsToSize(rewriter, indexingSizes, options);
+
+  // For each dimension in the operand's shape, iterate over indexingSizes and
+  // add
+  for (const auto &enResults : enumerate(indexingMap.getResults())) {
+    int64_t resultIndex = enResults.index();
+    AffineMap partialIndexingMap = indexingMap.getSubMap(
+        ArrayRef<unsigned>{static_cast<unsigned>(resultIndex)});
+
+    LLVM_DEBUG(DBGS() << "----resultIndex: " << resultIndex
+                      << " with partialIndexingMap: " << partialIndexingMap
+                      << "\n");
+
+    // Find all padding dimensions that contribute to this operand dimension
+    // and compute the padded term contribution to the final padded shape.
+    SmallVector<OpFoldResult> terms;
+    for (const auto &[paddingDim, paddingSize] : dimsToSize) {
+      LLVM_DEBUG(DBGS() << "------try apply padding of dim: " << paddingDim
+                        << " to: " << paddingSize << "\n");
+      if (!enResults.value().isFunctionOfDim(paddingDim))
+        continue;
+
+      LLVM_DEBUG(DBGS() << "------apply padding of dim: " << paddingDim
+                        << " to: " << paddingSize << "\n");
+
+      // Project non-'paddingDim' dimensions and compress the result.
+      llvm::SmallBitVector projectedDims(partialIndexingMap.getNumDims(), true);
+      projectedDims.flip(paddingDim);
+      AffineMap projectedMap =
+          mlir::projectDims(partialIndexingMap, projectedDims,
+                            /*compressDims=*/true);
+
+      // If we are padding to the next multiple of, compose with ceil(sz) * sz.
+      if (options.padToMultipleOf) {
+        AffineExpr d0, s0;
+        bindDims(rewriter.getContext(), d0);
+        bindSymbols(rewriter.getContext(), s0);
+        AffineMap ceilMap = AffineMap::get(1, 1, d0.ceilDiv(s0) * s0);
+        AffineMap composedMap = projectedMap.compose(ceilMap);
+        OpFoldResult paddingDimOfr = affine::makeComposedFoldedAffineApply(
+            rewriter, loc, composedMap,
+            {indexingSizes[paddingDim], paddingSize});
+        terms.push_back(paddingDimOfr);
+      } else {
+        // Otherwise just set to paddingSize.
+        OpFoldResult paddingDimOfr = affine::makeComposedFoldedAffineApply(
+            rewriter, loc, projectedMap, paddingSize);
+        terms.push_back(paddingDimOfr);
+      }
+
+      LLVM_DEBUG(DBGS() << "------new term: " << terms.back() << "\n");
+    }
+
+    // If there are no terms, just return the dim.
+    if (terms.empty()) {
+      paddedShape[resultIndex] =
+          createFoldedDimOp(rewriter, loc, v, resultIndex);
+      continue;
+    }
+
+    // Sum individual terms' contributions.
+    SmallVector<AffineExpr> dims(terms.size());
+    bindDimsList(rewriter.getContext(), MutableArrayRef{dims});
+    AffineExpr sumExpr = dims.front();
+    for (unsigned i = 1; i < dims.size(); ++i)
+      sumExpr = sumExpr + dims[i];
+    OpFoldResult paddedDimOfr =
+        affine::makeComposedFoldedAffineApply(rewriter, loc, sumExpr, terms);
+    paddedShape[resultIndex] = paddedDimOfr;
+  }
+
+  return paddedShape;
+}
+
+FailureOr<SmallVector<OpFoldResult>> linalg::computeLinalgPaddedShape(
+    RewriterBase &rewriter, OpOperand &operandToPad,
+    ArrayRef<Range> iterationDomain, const PadTilingInterfaceOptions &options) {
+  auto linalgOp = llvm::dyn_cast<LinalgOp>(operandToPad.getOwner());
+  if (!linalgOp)
+    return failure();
+
+  // clang-format off
+  assert(llvm::all_of(iterationDomain, [&rewriter](Range r) {
+    return r.offset == OpFoldResult(rewriter.getIndexAttr(0)) &&
+    r.stride == OpFoldResult(rewriter.getIndexAttr(1));
+  }) && "expected 0-offset 1-stride loop ranges");
+  // clang-format on
+  SmallVector<OpFoldResult> loopUpperBounds;
+  loopUpperBounds.reserve(iterationDomain.size());
+  for (const Range &range : iterationDomain)
+    loopUpperBounds.push_back(range.size);
+
+  AffineMap indexingMap = linalgOp.getMatchingIndexingMap(&operandToPad);
+  return computePaddedShape(
+      rewriter, cast<TypedValue<RankedTensorType>>(operandToPad.get()),
+      indexingMap, loopUpperBounds, options);
+}
+
+/// Pad a single operand to `paddedShape` using `paddingValueAttr` as padding
+/// Value.
+static Value padOperand(RewriterBase &rewriter, TilingInterface opToPad,
+                        TypedValue<RankedTensorType> v,
+                        ArrayRef<OpFoldResult> paddedShape,
+                        Attribute paddingValueAttr) {
+  Value paddingValue;
+  if (auto complexTy =
+          dyn_cast<ComplexType>(getElementTypeOrSelf(v.getType()))) {
+    auto complexAttr = cast<ArrayAttr>(paddingValueAttr);
+    paddingValue = rewriter.create<complex::ConstantOp>(opToPad.getLoc(),
+                                                        complexTy, complexAttr);
+  } else {
+    paddingValue = rewriter.create<arith::ConstantOp>(
+        opToPad.getLoc(), cast<TypedAttr>(paddingValueAttr));
+  }
+
+  // Pad the operand to the bounding box defined by `paddedShape`.
+  SmallVector<int64_t> tensorShape;
+  SmallVector<Value> dynDims;
+  for (OpFoldResult ofr : paddedShape) {
+    std::optional<int64_t> cst = getConstantIntValue(ofr);
+    tensorShape.push_back(cst.has_value() ? *cst : ShapedType::kDynamic);
+    if (!cst.has_value())
+      dynDims.push_back(ofr.dyn_cast<Value>());
+  }
+  // TODO: use dispatchIndexOpFoldResults(paddedShape, dynDims, paddedShape);
+
+  auto paddedTensorType =
+      RankedTensorType::get(tensorShape, getElementTypeOrSelf(v));
+  LLVM_DEBUG(DBGS() << "--SUCCESS, makeComposedPadHighOp with type: "
+                    << paddedTensorType);
+  return makeComposedPadHighOp(rewriter, opToPad.getLoc(), paddedTensorType, v,
+                               paddingValue, /*nofold=*/false, dynDims);
+}
+
+FailureOr<TilingInterface>
+linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
+                          const PadTilingInterfaceOptions &constOptions,
+                          SmallVector<tensor::PadOp> &padOps,
+                          PadSizeComputationFunction computePaddingSizeFun) {
+  LLVM_DEBUG(DBGS() << "Start rewriteAsPaddedOp : " << opToPad << "\n");
+  assert(constOptions.paddingSizes.size() ==
+             constOptions.paddingDimensions.size() &&
+         "invalid number of elements in padToMultipleOf");
+
+  Location loc = opToPad.getLoc();
+  PadTilingInterfaceOptions options(constOptions);
+  // Allow inference of pad values if they are not explicitly specified.
+  // TODO: be mindful about the value depending on the actual operation.
+  if (options.paddingValues.empty()) {
+    SmallVector<Type> types(opToPad->getOperandTypes());
+    llvm::append_range(types, opToPad->getResultTypes());
+    for (Type t : types) {
+      options.paddingValues.push_back(
+          rewriter.getZeroAttr(getElementTypeOrSelf(t)));
+    }
+  }
+
+  if (llvm::any_of(opToPad->getOperands(),
+                   [](Value v) { return isa<MemRefType>(v.getType()); })) {
+    return rewriter.notifyMatchFailure(opToPad,
+                                       "expected operation on tensors");
+  }
+
+  OpBuilder::InsertionGuard g(rewriter);
+  // Set IP after opToPad because we also take the dims of opToPad's output.
+  rewriter.setInsertionPointAfter(opToPad);
+
+  // 1. Get the loopUpperBounds from the TilingInterface.
+  SmallVector<Range> iterationDomain = opToPad.getIterationDomain(rewriter);
+
+  // 2. For each operand.
+  SmallVector<Value> newOperands;
+  newOperands.reserve(opToPad->getNumOperands());
+  for (OpOperand &opOperand : opToPad->getOpOperands()) {
+    LLVM_DEBUG(DBGS() << "--start padding oprd: " << opOperand.get() << "\n");
+    // 2.a. Compute padded shape.
+    FailureOr<SmallVector<OpFoldResult>> maybePaddedShape =
+        computePaddingSizeFun(rewriter, opOperand, iterationDomain, options);
+    if (failed(maybePaddedShape)) {
+      return rewriter.notifyMatchFailure(opToPad, "could not pad op");
+    }
+
+    // 2.b. Expect proper `paddingValues`.
+    // TODO: we may want to allow garbage padding in the future, in which case
+    // we would just not assert.
+    assert(opOperand.getOperandNumber() < options.paddingValues.size() &&
+           "--no padding value specified");
+    Attribute paddingValueAttr =
+        options.paddingValues[opOperand.getOperandNumber()];
+
+    // 2.c. Perform actual padding.
+    Value paddedOperand = padOperand(
+        rewriter, opToPad, cast<TypedValue<RankedTensorType>>(opOperand.get()),
+        *maybePaddedShape, paddingValueAttr);
+    LLVM_DEBUG(DBGS() << "--done padding operand: " << paddedOperand << "\n");
+
+    // 2.d. Perform actual padding.
+    newOperands.push_back(paddedOperand);
+    if (auto padOp = paddedOperand.getDefiningOp<tensor::PadOp>())
+      padOps.push_back(padOp);
+  }
+
+  // 3. Form the resulting tensor::ExtractSliceOp.
+  ReifiedRankedShapedTypeDims reifiedResultShapes;
+  if (failed(reifyResultShapes(rewriter, opToPad, reifiedResultShapes))) {
+    LLVM_DEBUG(DBGS() << "--failed to reify result shapes -> FAIL\n");
+    return rewriter.notifyMatchFailure(opToPad,
+                                       "failed to reify result shapes");
+  }
+  assert(reifiedResultShapes.size() == opToPad->getNumResults() &&
+         "expected same number of results");
+
+  // Clone `opToPad` to operate on the statically padded shapes.
+  auto resultTensorTypes =
+      ValueRange(newOperands).take_back(opToPad->getNumResults()).getTypes();
+  // clone **should** properly notify the rewriter.
+  TilingInterface paddedOp =
+      clone(rewriter, opToPad, resultTensorTypes, newOperands);
+  LLVM_DEBUG(DBGS() << "--cloned padded op: " << paddedOp << "\n");
+
+  // Recover the slice out of the new static results. This keeps the original
+  // opToPad around because it uses the dims of the original results.
+  SmallVector<Value> paddedSubtensorResults;
+  paddedSubtensorResults.reserve(opToPad->getNumResults());
+  for (const auto &en : llvm::enumerate(paddedOp->getResults())) {
+    Value paddedResult = en.value();
+    int64_t resultNumber = en.index();
+    int64_t rank = cast<RankedTensorType>(paddedResult.getType()).getRank();
+    SmallVector<OpFoldResult> offsets(rank, rewriter.getIndexAttr(0));
+    SmallVector<OpFoldResult> strides(rank, rewriter.getIndexAttr(1));
+    paddedSubtensorResults.push_back(rewriter.create<tensor::ExtractSliceOp>(
+        loc, paddedResult, offsets, reifiedResultShapes[resultNumber],
+        strides));
+  }
+
+  rewriter.replaceOp(opToPad, paddedSubtensorResults);
+
+  return paddedOp;
+}
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
new file mode 100644
index 000000000000..d502150e9e34
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
@@ -0,0 +1,138 @@
+// RUN: mlir-opt --transform-interpreter -canonicalize -split-input-file --verify-diagnostics %s | FileCheck %s
+
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<(d0) -> (-d0 + 24, 5)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<(d0) -> (-d0 + (d0 ceildiv 8) * 8)>
+
+//     CHECK-LABEL: pad_lhs
+func.func @pad_lhs(
+  %arg0: tensor<24x12xf32>, %arg1: tensor<12x25xf32>, %arg2: tensor<24x25xf32>)
+     -> tensor<24x25xf32>
+{
+  //      CHECK: scf.for %{{.*}} -> (tensor<24x25xf32>)
+  //      CHECK:   %[[MIN:.*]] = affine.min #[[$MAP0]](%{{.*}})
+  //      CHECK:   %[[H0:.*]] = affine.apply #[[$MAP1]](%[[MIN]])
+  //      CHECK:   tensor.pad %{{.*}} low[0, 0] high[%[[H0]], 0]
+  //      CHECK:     : tensor<?x12xf32> to tensor<?x12xf32>
+
+  //      CHECK:   %[[H1:.*]] = affine.apply #[[$MAP1]](%[[MIN]])
+  //      CHECK:   tensor.pad %{{.*}} low[0, 0] high[%[[H1]], 0]
+  //      CHECK:     : tensor<?x25xf32> to tensor<?x25xf32>
+
+  //      CHECK:   linalg.matmul ins(%{{.*}}, %{{.*}} : tensor<?x12xf32>, tensor<12x25xf32>) outs(%{{.*}} : tensor<?x25xf32>) -> tensor<?x25xf32>
+  
+  //      CHECK:   tensor.extract_slice %{{.*}}[0, 0] [%{{.*}}, 25] [1, 1]
+  //      CHECK:     : tensor<?x25xf32> to tensor<?x25xf32>
+  //      CHECK:   tensor.insert_slice %{{.*}} into %{{.*}}[%{{.*}}, 0] [%{{.*}}, 25] [1, 1]
+  // CHECK-SAME:     : tensor<?x25xf32> into tensor<24x25xf32>
+  %0 = linalg.matmul ins(%arg0, %arg1 : tensor<24x12xf32>, tensor<12x25xf32>) outs(%arg2 : tensor<24x25xf32>) -> tensor<24x25xf32>
+  func.return %0 : tensor<24x25xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+
+    // Tile to 5 then pad to 8 (supposedly to better hit vector ops).
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_padded, %_ = transform.structured.pad_tiling_interface %matmul_l1 to padding_sizes [8] pad_to_multiple_of {
+      padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
+      padding_dimensions=[0]
+    } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+    transform.yield
+  }
+}
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2, d0 + d1)>
+
+module {
+
+// CHECK-LABEL: @generic
+// CHECK-SAME:      %[[T0:.*]]: tensor<7x5xf32>,
+// CHECK-SAME:      %[[T1:.*]]: tensor<7x11x12xf32>)
+  func.func @generic(%arg0: tensor<7x5xf32>, %arg1: tensor<7x11x12xf32>) -> tensor<7x11x12xf32> {
+
+  //  CHECK-DAG: %[[CST:.*]] = arith.constant 0.
+
+  //      CHECK: %[[PAD0:.*]] = tensor.pad %[[T0]] low[0, 0] high[2, 0]
+  //      CHECK:   : tensor<7x5xf32> to tensor<9x5xf32>
+  //      CHECK: %[[PAD1:.*]] = tensor.pad %[[T1]] low[0, 0, 0] high[2, 4, 2] {
+  //      CHECK:   : tensor<7x11x12xf32> to tensor<9x15x14xf32>
+  // CHECK-NEXT: linalg.generic
+  //      CHECK: tensor.extract_slice %{{.*}}[0, 0, 0] [7, 11, 12] [1, 1, 1] : tensor<9x15x14xf32> to tensor<7x11x12xf32>
+  %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0 : tensor<7x5xf32>) outs(%arg1 : tensor<7x11x12xf32>) {
+    ^bb0(%in: f32, %out: f32):
+      linalg.yield %in : f32
+    } -> tensor<7x11x12xf32>
+    return %0 : tensor<7x11x12xf32>
+  }
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+      %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [3, 5] pad_to_multiple_of {
+        padding_dimensions = [0, 2], 
+        padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
+      } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield 
+    }
+  }
+}
+
+// -----
+
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0, s1] -> (-s1 + (s0 ceildiv 3) * 3)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0, s1] -> (-s1 + (s0 ceildiv 3) * 3 + 5)>
+// CHECK-DAG: #[[$MAP2:.*]] = affine_map<()[s0] -> (s0 + 5)>
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2, d0 + d1)>
+module {
+
+// CHECK-LABEL: @generic
+// CHECK-SAME:      %[[T0:.*]]: tensor<?x5xf32>,
+// CHECK-SAME:      %[[T1:.*]]: tensor<?x11x?xf32>)
+  func.func @generic(%arg0: tensor<?x5xf32>, %arg1: tensor<?x11x?xf32>) -> tensor<?x11x?xf32> {
+
+  //  CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+  //  CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+  //
+  //      CHECK: %[[D0_0:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x5xf32>
+  //      CHECK: %[[D0_1:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x5xf32>
+  //      CHECK: %[[H0:.*]] = affine.apply #[[$MAP0]]()[%[[D0_0]], %[[D0_1]]]
+  //      CHECK: tensor.pad %{{.*}} low[0, 0] high[%[[H0]], 0] {
+  //      CHECK:   : tensor<?x5xf32> to tensor<?x5xf32>
+  //
+  //      CHECK: %[[D0_2:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x11x?xf32>
+  //      CHECK: %[[H1:.*]] = affine.apply #[[$MAP0]]()[%[[D0_0]], %[[D0_2]]]
+  //      CHECK: %[[D2_0:.*]] = tensor.dim %{{.*}}, %[[C2]] : tensor<?x11x?xf32>
+  //      CHECK: %[[H2:.*]] = affine.apply #[[$MAP1]]()[%[[D0_0]], %[[D2_0]]]
+  //      CHECK: tensor.pad %{{.*}} low[0, 0, 0] high[%[[H1]], 4, %[[H2]]] {
+  //      CHECK:   : tensor<?x11x?xf32> to tensor<?x15x?xf32>
+  //
+  //      CHECK: %[[D0_3:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x5xf32>
+  //      CHECK: %[[D2_1:.*]] = affine.apply #[[$MAP2]]()[%[[D0_3]]]
+  //      CHECK: linalg.generic {{.*}} ins(%{{.*}} : tensor<?x5xf32>) outs(%{{.*}} : tensor<?x15x?xf32>) {
+  //      CHECK: } -> tensor<?x15x?xf32>
+  //      CHECK: tensor.extract_slice %{{.*}}[0, 0, 0] [%[[D0_3]], 11, %[[D2_1]]] [1, 1, 1] : tensor<?x15x?xf32> to tensor<?x11x?xf32>
+  //
+  %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0 : tensor<?x5xf32>) outs(%arg1 : tensor<?x11x?xf32>) {
+    ^bb0(%in: f32, %out: f32):
+      linalg.yield %in : f32
+    } -> tensor<?x11x?xf32>
+    return %0 : tensor<?x11x?xf32>
+  }
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+      %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [3, 5] pad_to_multiple_of {
+        padding_dimensions = [0, 2], 
+        padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
+      } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield 
+    }
+  }
+}
\ No newline at end of file
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
new file mode 100644
index 000000000000..e2b97cf908a8
--- /dev/null
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
@@ -0,0 +1,129 @@
+// RUN: mlir-opt --transform-interpreter -canonicalize -split-input-file --verify-diagnostics %s | FileCheck %s
+
+//     CHECK-LABEL: pad_lhs
+func.func @pad_lhs(
+  %arg0: tensor<24x12xf32>, %arg1: tensor<12x25xf32>, %arg2: tensor<24x25xf32>)
+     -> tensor<24x25xf32>
+{
+  //      CHECK: scf.for %{{.*}} -> (tensor<24x25xf32>)
+  //      CHECK:   tensor.pad %{{.*}} 
+  //      CHECK:     : tensor<?x12xf32> to tensor<8x12xf32>
+  //      CHECK:   tensor.pad %{{.*}} 
+  //      CHECK:     : tensor<?x25xf32> to tensor<8x25xf32>
+  //      CHECK:   linalg.matmul ins(%{{.*}}, %{{.*}} : tensor<8x12xf32>, tensor<12x25xf32>) outs(%{{.*}} : tensor<8x25xf32>) -> tensor<8x25xf32>
+  //      CHECK:   tensor.extract_slice %{{.*}}[0, 0] [%{{.*}}, 25] [1, 1]
+  //      CHECK:     : tensor<8x25xf32> to tensor<?x25xf32>
+  //      CHECK:   tensor.insert_slice %{{.*}} into %{{.*}}[%{{.*}}, 0] [%{{.*}}, 25] [1, 1]
+  // CHECK-SAME:     : tensor<?x25xf32> into tensor<24x25xf32>
+  %0 = linalg.matmul ins(%arg0, %arg1 : tensor<24x12xf32>, tensor<12x25xf32>) outs(%arg2 : tensor<24x25xf32>) -> tensor<24x25xf32>
+  func.return %0 : tensor<24x25xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %matmul = transform.structured.match ops{["linalg.matmul"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+
+    // Tile to 5 then pad to 8 (supposedly to better hit vector ops).
+    %matmul_l1, %loops_l1 = transform.structured.tile_using_for %matmul tile_sizes [5] : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+    %matmul_padded, %_ = transform.structured.pad_tiling_interface %matmul_l1 to padding_sizes [8] {
+      padding_values=[0.0: f32, 0.0 : f32, 0.0 : f32],
+      padding_dimensions=[0]
+    } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+    transform.yield
+  }
+}
+
+// -----
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2, d0 + d1)>
+module {
+
+// CHECK-LABEL: @generic
+// CHECK-SAME:      %[[T0:.*]]: tensor<7x5xf32>,
+// CHECK-SAME:      %[[T1:.*]]: tensor<7x11x12xf32>)
+  func.func @generic(%arg0: tensor<7x5xf32>, %arg1: tensor<7x11x12xf32>) -> tensor<7x11x12xf32> {
+
+  //  CHECK-DAG: %[[CST:.*]] = arith.constant 0.
+
+  //      CHECK: %[[PAD0:.*]] = tensor.pad %[[T0]] low[0, 0] high[1, 0]
+  //      CHECK:   : tensor<7x5xf32> to tensor<8x5xf32>
+  //      CHECK: %[[PAD1:.*]] = tensor.pad %[[T1]] low[0, 0, 0] high[1, 3, 1] {
+  //      CHECK:   : tensor<7x11x12xf32> to tensor<8x14x13xf32>
+  // CHECK-NEXT: linalg.generic
+  //      CHECK: tensor.extract_slice %{{.*}}[0, 0, 0] [7, 11, 12] [1, 1, 1] : tensor<8x14x13xf32> to tensor<7x11x12xf32>
+  %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0 : tensor<7x5xf32>) outs(%arg1 : tensor<7x11x12xf32>) {
+    ^bb0(%in: f32, %out: f32):
+      linalg.yield %in : f32
+    } -> tensor<7x11x12xf32>
+    return %0 : tensor<7x11x12xf32>
+  }
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+      %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [8, 14] {
+        padding_dimensions = [0, 2], 
+        padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
+      } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield 
+    }
+  }
+}
+
+
+// -----
+
+
+// CHECK-DAG: #[[$MAP0:.*]] = affine_map<()[s0] -> (-s0 + 8)>
+// CHECK-DAG: #[[$MAP1:.*]] = affine_map<()[s0] -> (-s0 + 13)>
+// CHECK-DAG: #[[$MAP2:.*]] = affine_map<()[s0] -> (s0 + 5)>
+
+#map = affine_map<(d0, d1, d2) -> (d0, d1)>
+#map1 = affine_map<(d0, d1, d2) -> (d0, d2, d0 + d1)>
+module {
+
+// CHECK-LABEL: @generic
+// CHECK-SAME:      %[[T0:.*]]: tensor<?x5xf32>,
+// CHECK-SAME:      %[[T1:.*]]: tensor<?x11x?xf32>)
+  func.func @generic(%arg0: tensor<?x5xf32>, %arg1: tensor<?x11x?xf32>) -> tensor<?x11x?xf32> {
+
+  //  CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
+  //  CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index
+  //
+  //      CHECK: %[[D0_0:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x5xf32>
+  //      CHECK: %[[H0:.*]] = affine.apply #[[$MAP0]]()[%[[D0_0]]]
+  //      CHECK: tensor.pad %{{.*}} low[0, 0] high[%[[H0]], 0] {
+  //      CHECK:   : tensor<?x5xf32> to tensor<8x5xf32>
+  //
+  //      CHECK: %[[D0_1:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x11x?xf32>
+  //      CHECK: %[[H1:.*]] = affine.apply #[[$MAP0]]()[%[[D0_1]]]
+  //      CHECK: %[[D2_0:.*]] = tensor.dim %{{.*}}, %[[C2]] : tensor<?x11x?xf32>
+  //      CHECK: %[[H2:.*]] = affine.apply #[[$MAP1]]()[%[[D2_0]]]
+  //      CHECK: tensor.pad %{{.*}} low[0, 0, 0] high[%[[H1]], 3, %[[H2]]] {
+  //      CHECK:   : tensor<?x11x?xf32> to tensor<8x14x13xf32>
+  //
+  //      CHECK: %[[D0_2:.*]] = tensor.dim %{{.*}}, %[[C0]] : tensor<?x5xf32>
+  //      CHECK: %[[D2_1:.*]] = affine.apply #[[$MAP2]]()[%[[D0_2]]]
+  //      CHECK: linalg.generic {{.*}} ins(%{{.*}} : tensor<8x5xf32>) outs(%{{.*}} : tensor<8x14x13xf32>) {
+  //      CHECK: } -> tensor<8x14x13xf32>
+  //      CHECK: tensor.extract_slice %{{.*}}[0, 0, 0] [%[[D0_2]], 11, %[[D2_1]]] [1, 1, 1] : tensor<8x14x13xf32> to tensor<?x11x?xf32>
+  //
+  %0 = linalg.generic {indexing_maps = [#map, #map1], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg0 : tensor<?x5xf32>) outs(%arg1 : tensor<?x11x?xf32>) {
+    ^bb0(%in: f32, %out: f32):
+      linalg.yield %in : f32
+    } -> tensor<?x11x?xf32>
+    return %0 : tensor<?x11x?xf32>
+  }
+  module attributes {transform.with_named_sequence} {
+    transform.named_sequence @__transform_main(%arg0: !transform.any_op {transform.readonly}) {
+      %0 = transform.structured.match ops{["linalg.generic"]} in %arg0 : (!transform.any_op) -> !transform.any_op
+      %padded, %pad = transform.structured.pad_tiling_interface %0 to padding_sizes [8, 14] {
+        padding_dimensions = [0, 2], 
+        padding_values = [0.000000e+00 : f32, 0.000000e+00 : f32, 0.000000e+00 : f32]
+      } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+      transform.yield 
+    }
+  }
+}
\ No newline at end of file
-- 
cgit v1.2.3


From 7af545237f8509d40b9ab1dd6526210e09bc76bf Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <Nico.Vasilache@amd.com>
Date: Fri, 20 Jun 2025 12:33:14 +0200
Subject: [NFC] Update transform-op-pad-tiling-interface-multiple-of.mlir

Missing NL
---
 .../Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
index d502150e9e34..5ac35c14be3f 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface-multiple-of.mlir
@@ -135,4 +135,4 @@ module {
       transform.yield 
     }
   }
-}
\ No newline at end of file
+}
-- 
cgit v1.2.3


From 227cd56e1330e51bc48f4fd71fabfc203b7ca31d Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <Nico.Vasilache@amd.com>
Date: Fri, 20 Jun 2025 12:33:56 +0200
Subject: [NFC] Update transform-op-pad-tiling-interface.mlir

Missing NL
---
 mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
index e2b97cf908a8..c361885693cb 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
@@ -126,4 +126,4 @@ module {
       transform.yield 
     }
   }
-}
\ No newline at end of file
+}
-- 
cgit v1.2.3


From 68732ce8e01938227378b4e6f7850ba85c978726 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 20 Jun 2025 11:46:25 +0100
Subject: [LLVM][CodeGen][SVE] Add isel for bfloat unordered reductions.
 (#143540)

The omissions are VECREDUCE_SEQ_* and MUL. The former goes down a
different code path and the latter is unsupported across all element types.
---
 .../lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp |  45 +++-
 llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp   |  15 +-
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    |   4 +-
 llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll   | 279 +++++++++++++++++++++
 4 files changed, 328 insertions(+), 15 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 4a1cd642233e..f908a66128ec 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -189,6 +189,12 @@ class VectorLegalizer {
 
   void PromoteSTRICT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
 
+  /// Calculate the reduction using a type of higher precision and round the
+  /// result to match the original type. Setting NonArithmetic signifies the
+  /// rounding of the result does not affect its value.
+  void PromoteFloatVECREDUCE(SDNode *Node, SmallVectorImpl<SDValue> &Results,
+                             bool NonArithmetic);
+
 public:
   VectorLegalizer(SelectionDAG& dag) :
       DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
@@ -500,20 +506,14 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::VECREDUCE_UMAX:
   case ISD::VECREDUCE_UMIN:
   case ISD::VECREDUCE_FADD:
-  case ISD::VECREDUCE_FMUL:
-  case ISD::VECTOR_FIND_LAST_ACTIVE:
-    Action = TLI.getOperationAction(Node->getOpcode(),
-                                    Node->getOperand(0).getValueType());
-    break;
   case ISD::VECREDUCE_FMAX:
-  case ISD::VECREDUCE_FMIN:
   case ISD::VECREDUCE_FMAXIMUM:
+  case ISD::VECREDUCE_FMIN:
   case ISD::VECREDUCE_FMINIMUM:
+  case ISD::VECREDUCE_FMUL:
+  case ISD::VECTOR_FIND_LAST_ACTIVE:
     Action = TLI.getOperationAction(Node->getOpcode(),
                                     Node->getOperand(0).getValueType());
-    // Defer non-vector results to LegalizeDAG.
-    if (Action == TargetLowering::Promote)
-      Action = TargetLowering::Legal;
     break;
   case ISD::VECREDUCE_SEQ_FADD:
   case ISD::VECREDUCE_SEQ_FMUL:
@@ -688,6 +688,24 @@ void VectorLegalizer::PromoteSTRICT(SDNode *Node,
   Results.push_back(Round.getValue(1));
 }
 
+void VectorLegalizer::PromoteFloatVECREDUCE(SDNode *Node,
+                                            SmallVectorImpl<SDValue> &Results,
+                                            bool NonArithmetic) {
+  MVT OpVT = Node->getOperand(0).getSimpleValueType();
+  assert(OpVT.isFloatingPoint() && "Expected floating point reduction!");
+  MVT NewOpVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OpVT);
+
+  SDLoc DL(Node);
+  SDValue NewOp = DAG.getNode(ISD::FP_EXTEND, DL, NewOpVT, Node->getOperand(0));
+  SDValue Rdx =
+      DAG.getNode(Node->getOpcode(), DL, NewOpVT.getVectorElementType(), NewOp,
+                  Node->getFlags());
+  SDValue Res =
+      DAG.getNode(ISD::FP_ROUND, DL, Node->getValueType(0), Rdx,
+                  DAG.getIntPtrConstant(NonArithmetic, DL, /*isTarget=*/true));
+  Results.push_back(Res);
+}
+
 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
   // For a few operations there is a specific concept for promotion based on
   // the operand's type.
@@ -719,6 +737,15 @@ void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
   case ISD::STRICT_FMA:
     PromoteSTRICT(Node, Results);
     return;
+  case ISD::VECREDUCE_FADD:
+    PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/false);
+    return;
+  case ISD::VECREDUCE_FMAX:
+  case ISD::VECREDUCE_FMAXIMUM:
+  case ISD::VECREDUCE_FMIN:
+  case ISD::VECREDUCE_FMINIMUM:
+    PromoteFloatVECREDUCE(Node, Results, /*NonArithmetic=*/true);
+    return;
   case ISD::FP_ROUND:
   case ISD::FP_EXTEND:
     // These operations are used to do promotion so they can't be promoted
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 52f19cc6e1ab..1e470318ced0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -11413,13 +11413,9 @@ SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
   SDValue Op = Node->getOperand(0);
   EVT VT = Op.getValueType();
 
-  if (VT.isScalableVector())
-    report_fatal_error(
-        "Expanding reductions for scalable vectors is undefined.");
-
   // Try to use a shuffle reduction for power of two vectors.
   if (VT.isPow2VectorType()) {
-    while (VT.getVectorNumElements() > 1) {
+    while (VT.getVectorElementCount().isKnownMultipleOf(2)) {
       EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
       if (!isOperationLegalOrCustom(BaseOpcode, HalfVT))
         break;
@@ -11428,9 +11424,18 @@ SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const {
       std::tie(Lo, Hi) = DAG.SplitVector(Op, dl);
       Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi, Node->getFlags());
       VT = HalfVT;
+
+      // Stop if splitting is enough to make the reduction legal.
+      if (isOperationLegalOrCustom(Node->getOpcode(), HalfVT))
+        return DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Op,
+                           Node->getFlags());
     }
   }
 
+  if (VT.isScalableVector())
+    reportFatalInternalError(
+        "Expanding reductions for scalable vectors is undefined.");
+
   EVT EltVT = VT.getVectorElementType();
   unsigned NumElts = VT.getVectorNumElements();
 
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 8f208c4d006c..eaaff0529cbd 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1762,7 +1762,9 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
 
     for (auto Opcode :
          {ISD::FCEIL, ISD::FDIV, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
-          ISD::FROUND, ISD::FROUNDEVEN, ISD::FSQRT, ISD::FTRUNC, ISD::SETCC}) {
+          ISD::FROUND, ISD::FROUNDEVEN, ISD::FSQRT, ISD::FTRUNC, ISD::SETCC,
+          ISD::VECREDUCE_FADD, ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMAXIMUM,
+          ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMINIMUM}) {
       setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
       setOperationPromotedToType(Opcode, MVT::nxv4bf16, MVT::nxv4f32);
       setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll
new file mode 100644
index 000000000000..7f79c9c5431e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll
@@ -0,0 +1,279 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mattr=+sve,+bf16            < %s | FileCheck %s -check-prefixes=CHECK,SVE
+; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s -check-prefixes=CHECK,SME
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; FADDV
+
+define bfloat @faddv_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: faddv_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call fast bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat zeroinitializer, <vscale x 2 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @faddv_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: faddv_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat zeroinitializer, <vscale x 4 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @faddv_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: faddv_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpkhi z1.s, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    fadd z0.s, z0.s, z1.s
+; CHECK-NEXT:    faddv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat zeroinitializer, <vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
+; FMAXNMV
+
+define bfloat @fmaxv_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: fmaxv_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmax.nxv2bf16(<vscale x 2 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fmaxv_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: fmaxv_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fmaxv_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: fmaxv_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpkhi z1.s, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    fmaxnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
+; FMINNMV
+
+define bfloat @fminv_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: fminv_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmin.nxv2bf16(<vscale x 2 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fminv_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: fminv_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fminnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fminv_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: fminv_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpkhi z1.s, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    fminnm z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    fminnmv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
+; FMAXV
+
+define bfloat @fmaximumv_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: fmaximumv_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fmaxv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmaximum.nxv2bf16(<vscale x 2 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fmaximumv_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: fmaximumv_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fmaximumv_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: fmaximumv_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpkhi z1.s, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    fmax z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    fmaxv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fmaximum.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
+; FMINV
+
+define bfloat @fminimumv_nxv2bf16(<vscale x 2 x bfloat> %a) {
+; CHECK-LABEL: fminimumv_nxv2bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    fminv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fminimum.nxv2bf16(<vscale x 2 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fminimumv_nxv4bf16(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: fminimumv_nxv4bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fminv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat> %a)
+  ret bfloat %res
+}
+
+define bfloat @fminimumv_nxv8bf16(<vscale x 8 x bfloat> %a) {
+; CHECK-LABEL: fminimumv_nxv8bf16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uunpkhi z1.s, z0.h
+; CHECK-NEXT:    uunpklo z0.s, z0.h
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    lsl z1.s, z1.s, #16
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    fmin z0.s, p0/m, z0.s, z1.s
+; CHECK-NEXT:    fminv s0, p0, z0.s
+; CHECK-NEXT:    bfcvt h0, s0
+; CHECK-NEXT:    ret
+  %res = call bfloat @llvm.vector.reduce.fminimum.nxv8bf16(<vscale x 8 x bfloat> %a)
+  ret bfloat %res
+}
+
+; The reduction is performed at a higher precision. Because add operations
+; can utilise that precision, its result must be rounded even if it's then
+; promoted.
+define float @promoted_fadd(<vscale x 4 x bfloat> %a) {
+; SVE-LABEL: promoted_fadd:
+; SVE:       // %bb.0:
+; SVE-NEXT:    lsl z0.s, z0.s, #16
+; SVE-NEXT:    ptrue p0.s
+; SVE-NEXT:    faddv s0, p0, z0.s
+; SVE-NEXT:    bfcvt h0, s0
+; SVE-NEXT:    shll v0.4s, v0.4h, #16
+; SVE-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; SVE-NEXT:    ret
+;
+; SME-LABEL: promoted_fadd:
+; SME:       // %bb.0:
+; SME-NEXT:    lsl z0.s, z0.s, #16
+; SME-NEXT:    ptrue p0.s
+; SME-NEXT:    faddv s0, p0, z0.s
+; SME-NEXT:    bfcvt h0, s0
+; SME-NEXT:    fmov w8, s0
+; SME-NEXT:    lsl w8, w8, #16
+; SME-NEXT:    fmov s0, w8
+; SME-NEXT:    ret
+  %rdx = call fast bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat zeroinitializer, <vscale x 4 x bfloat> %a)
+  %res = fpext bfloat %rdx to float
+  ret float %res
+}
+
+; The reduction is performed at a higher precision. Because min/max operations
+; don't utilise that precision, its result can be used directly.
+define float @promoted_fmax(<vscale x 4 x bfloat> %a) {
+; CHECK-LABEL: promoted_fmax:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    lsl z0.s, z0.s, #16
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    fmaxnmv s0, p0, z0.s
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $z0
+; CHECK-NEXT:    ret
+  %rdx = call bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat> %a)
+  %res = fpext bfloat %rdx to float
+  ret float %res
+}
+
+declare bfloat @llvm.vector.reduce.fadd.nxv2bf16(bfloat, <vscale x 2 x bfloat>)
+declare bfloat @llvm.vector.reduce.fadd.nxv4bf16(bfloat, <vscale x 4 x bfloat>)
+declare bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat, <vscale x 8 x bfloat>)
+
+declare bfloat @llvm.vector.reduce.fmax.nxv2bf16(<vscale x 2 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmax.nxv4bf16(<vscale x 4 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat>)
+
+declare bfloat @llvm.vector.reduce.fmin.nxv2bf16(<vscale x 2 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmin.nxv4bf16(<vscale x 4 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat>)
+
+declare bfloat @llvm.vector.reduce.fmaximum.nxv2bf16(<vscale x 2 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmaximum.nxv4bf16(<vscale x 4 x bfloat>)
+declare bfloat @llvm.vector.reduce.fmaximum.nxv8bf16(<vscale x 8 x bfloat>)
+
+declare bfloat @llvm.vector.reduce.fminimum.nxv2bf16(<vscale x 2 x bfloat>)
+declare bfloat @llvm.vector.reduce.fminimum.nxv4bf16(<vscale x 4 x bfloat>)
+declare bfloat @llvm.vector.reduce.fminimum.nxv8bf16(<vscale x 8 x bfloat>)
-- 
cgit v1.2.3


From 4ec6d127c1857e77d70236b15b03d23ba1283a3d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 11:43:01 +0100
Subject: [X86] movmsk-cmp.ll - regenerate VPTERNLOG asm comments

---
 llvm/test/CodeGen/X86/movmsk-cmp.ll | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/test/CodeGen/X86/movmsk-cmp.ll b/llvm/test/CodeGen/X86/movmsk-cmp.ll
index 9b624a935bad..2f8cd4d41af5 100644
--- a/llvm/test/CodeGen/X86/movmsk-cmp.ll
+++ b/llvm/test/CodeGen/X86/movmsk-cmp.ll
@@ -4098,7 +4098,7 @@ define i1 @movmsk_v8i16_var(<8 x i16> %x, <8 x i16> %y, i32 %z) {
 ; KNL-NEXT:    vpcmpgtw %xmm1, %xmm0, %xmm0
 ; KNL-NEXT:    vpmovsxwq %xmm0, %zmm0
 ; KNL-NEXT:    vptestmq %zmm0, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vpmovdw %zmm0, %ymm0
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $7, %edi
@@ -4143,7 +4143,7 @@ define i1 @movmsk_v4i32_var(<4 x i32> %x, <4 x i32> %y, i32 %z) {
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vpcmpgtd %zmm0, %zmm1, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $3, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,4), %eax
@@ -4200,7 +4200,7 @@ define i1 @movmsk_v2i64_var(<2 x i64> %x, <2 x i64> %y, i32 %z) {
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vpcmpneqq %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $1, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,8), %eax
@@ -4247,7 +4247,7 @@ define i1 @movmsk_v4f32_var(<4 x float> %x, <4 x float> %y, i32 %z) {
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vcmpeq_uqps %zmm1, %zmm0, %k1
-; KNL-NEXT:    vpternlogd $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $3, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,4), %eax
@@ -4291,7 +4291,7 @@ define i1 @movmsk_v2f64_var(<2 x double> %x, <2 x double> %y, i32 %z) {
 ; KNL-NEXT:    # kill: def $xmm1 killed $xmm1 def $zmm1
 ; KNL-NEXT:    # kill: def $xmm0 killed $xmm0 def $zmm0
 ; KNL-NEXT:    vcmplepd %zmm0, %zmm1, %k1
-; KNL-NEXT:    vpternlogq $255, %zmm0, %zmm0, %zmm0 {%k1} {z}
+; KNL-NEXT:    vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
 ; KNL-NEXT:    vmovdqa %xmm0, -{{[0-9]+}}(%rsp)
 ; KNL-NEXT:    andl $1, %edi
 ; KNL-NEXT:    movzbl -24(%rsp,%rdi,8), %eax
-- 
cgit v1.2.3


From 3e99aa6c0a36ec4d6f126882b1a06436767cbf73 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Fri, 20 Jun 2025 13:02:22 +0200
Subject: [PredicateInfo] Clean up DFS sorting (NFC) (#144943)

The comparison function for ValueDFS was confused in a number of ways.
Most significantly, it had a bunch of logic based on Def -- however, Def
is always null during sorting, it only gets set later. At this point
defs only have PInfo set.

Clean up the implementation to remove various dead code.
---
 llvm/lib/Transforms/Utils/PredicateInfo.cpp | 105 ++++++++++------------------
 1 file changed, 35 insertions(+), 70 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/PredicateInfo.cpp b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
index 9b239d9161e7..97f13e3b2674 100644
--- a/llvm/lib/Transforms/Utils/PredicateInfo.cpp
+++ b/llvm/lib/Transforms/Utils/PredicateInfo.cpp
@@ -92,19 +92,6 @@ struct ValueDFS {
   PredicateBase *PInfo = nullptr;
 };
 
-// Perform a strict weak ordering on instructions and arguments.
-static bool valueComesBefore(const Value *A, const Value *B) {
-  auto *ArgA = dyn_cast_or_null<Argument>(A);
-  auto *ArgB = dyn_cast_or_null<Argument>(B);
-  if (ArgA && !ArgB)
-    return true;
-  if (ArgB && !ArgA)
-    return false;
-  if (ArgA && ArgB)
-    return ArgA->getArgNo() < ArgB->getArgNo();
-  return cast<Instruction>(A)->comesBefore(cast<Instruction>(B));
-}
-
 // This compares ValueDFS structures. Doing so allows us to walk the minimum
 // number of instructions necessary to compute our def/use ordering.
 struct ValueDFS_Compare {
@@ -114,28 +101,34 @@ struct ValueDFS_Compare {
   bool operator()(const ValueDFS &A, const ValueDFS &B) const {
     if (&A == &B)
       return false;
-    // The only case we can't directly compare them is when they in the same
-    // block, and both have localnum == middle.  In that case, we have to use
-    // comesbefore to see what the real ordering is, because they are in the
-    // same basic block.
+    assert(!A.Def && !B.Def && "Should not have Def during comparison");
 
-    assert((A.DFSIn != B.DFSIn || A.DFSOut == B.DFSOut) &&
+    // Order by block first.
+    if (A.DFSIn != B.DFSIn)
+      return A.DFSIn < B.DFSIn;
+    assert(A.DFSOut == B.DFSOut &&
            "Equal DFS-in numbers imply equal out numbers");
-    bool SameBlock = A.DFSIn == B.DFSIn;
+
+    // Then order by first/middle/last.
+    if (A.LocalNum != B.LocalNum)
+      return A.LocalNum < B.LocalNum;
 
     // We want to put the def that will get used for a given set of phi uses,
     // before those phi uses.
     // So we sort by edge, then by def.
     // Note that only phi nodes uses and defs can come last.
-    if (SameBlock && A.LocalNum == LN_Last && B.LocalNum == LN_Last)
+    if (A.LocalNum == LN_Last)
       return comparePHIRelated(A, B);
 
-    bool isADef = A.Def;
-    bool isBDef = B.Def;
-    if (!SameBlock || A.LocalNum != LN_Middle || B.LocalNum != LN_Middle)
-      return std::tie(A.DFSIn, A.LocalNum, isADef) <
-             std::tie(B.DFSIn, B.LocalNum, isBDef);
-    return localComesBefore(A, B);
+    // Use block-local ordering for instructions in the middle.
+    if (A.LocalNum == LN_Middle)
+      return localComesBefore(A, B);
+
+    // The order of PredicateInfo definitions at the start of the block does not
+    // matter.
+    assert(A.LocalNum == LN_First);
+    assert(A.PInfo && B.PInfo && "Must be predicate info def");
+    return false;
   }
 
   // For a phi use, or a non-materialized def, return the edge it represents.
@@ -173,60 +166,32 @@ struct ValueDFS_Compare {
     DomTreeNode *DomBDest = DT.getNode(BDest);
     unsigned AIn = DomADest->getDFSNumIn();
     unsigned BIn = DomBDest->getDFSNumIn();
-    bool isADef = A.Def;
-    bool isBDef = B.Def;
-    assert((!A.Def || !A.U) && (!B.Def || !B.U) &&
+    bool isAUse = A.U;
+    bool isBUse = B.U;
+    assert((!A.PInfo || !A.U) && (!B.PInfo || !B.U) &&
            "Def and U cannot be set at the same time");
     // Now sort by edge destination and then defs before uses.
-    return std::tie(AIn, isADef) < std::tie(BIn, isBDef);
+    return std::tie(AIn, isAUse) < std::tie(BIn, isBUse);
   }
 
-  // Get the definition of an instruction that occurs in the middle of a block.
-  Value *getMiddleDef(const ValueDFS &VD) const {
-    if (VD.Def)
-      return VD.Def;
-    // It's possible for the defs and uses to be null.  For branches, the local
-    // numbering will say the placed predicaeinfos should go first (IE
-    // LN_beginning), so we won't be in this function. For assumes, we will end
-    // up here, beause we need to order the def we will place relative to the
-    // assume.  So for the purpose of ordering, we pretend the def is right
-    // after the assume, because that is where we will insert the info.
-    if (!VD.U) {
-      assert(VD.PInfo &&
-             "No def, no use, and no predicateinfo should not occur");
-      assert(isa<PredicateAssume>(VD.PInfo) &&
-             "Middle of block should only occur for assumes");
-      return cast<PredicateAssume>(VD.PInfo)->AssumeInst->getNextNode();
-    }
-    return nullptr;
-  }
+  const Instruction *getDefOrUser(const ValueDFS &VD) const {
+    if (VD.U)
+      return cast<Instruction>(VD.U->getUser());
 
-  // Return either the Def, if it's not null, or the user of the Use, if the def
-  // is null.
-  const Instruction *getDefOrUser(const Value *Def, const Use *U) const {
-    if (Def)
-      return cast<Instruction>(Def);
-    return cast<Instruction>(U->getUser());
+    // For the purpose of ordering, we pretend the def is right after the
+    // assume, because that is where we will insert the info.
+    assert(VD.PInfo && "No use, and no predicateinfo should not occur");
+    assert(isa<PredicateAssume>(VD.PInfo) &&
+           "Middle of block should only occur for assumes");
+    return cast<PredicateAssume>(VD.PInfo)->AssumeInst->getNextNode();
   }
 
   // This performs the necessary local basic block ordering checks to tell
   // whether A comes before B, where both are in the same basic block.
   bool localComesBefore(const ValueDFS &A, const ValueDFS &B) const {
-    auto *ADef = getMiddleDef(A);
-    auto *BDef = getMiddleDef(B);
-
-    // See if we have real values or uses. If we have real values, we are
-    // guaranteed they are instructions or arguments. No matter what, we are
-    // guaranteed they are in the same block if they are instructions.
-    auto *ArgA = dyn_cast_or_null<Argument>(ADef);
-    auto *ArgB = dyn_cast_or_null<Argument>(BDef);
-
-    if (ArgA || ArgB)
-      return valueComesBefore(ArgA, ArgB);
-
-    auto *AInst = getDefOrUser(ADef, A.U);
-    auto *BInst = getDefOrUser(BDef, B.U);
-    return valueComesBefore(AInst, BInst);
+    const Instruction *AInst = getDefOrUser(A);
+    const Instruction *BInst = getDefOrUser(B);
+    return AInst->comesBefore(BInst);
   }
 };
 
-- 
cgit v1.2.3


From cbd496581fb6953a9a8d8387a010cc3a67d4654b Mon Sep 17 00:00:00 2001
From: Chris Jackson <chris.jackson@amd.com>
Date: Fri, 20 Jun 2025 12:17:01 +0100
Subject: [NFC][AMDGPU] Automate any_extend_vector_inreg.ll check line
 generation (#145013)

Convert the test to use update_llc_test_checks.py.
---
 .../test/CodeGen/AMDGPU/any_extend_vector_inreg.ll | 167 ++++++++++++++++++---
 1 file changed, 143 insertions(+), 24 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
index 8bcef24c8e23..cc9f595f9d0b 100644
--- a/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/any_extend_vector_inreg.ll
@@ -1,30 +1,149 @@
-; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
-; RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -mtriple=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
 
-; GCN-LABEL: {{^}}any_extend_vector_inreg_v16i8_to_v4i32:
-; GCN: s_load_dwordx8
-; GCN-DAG: s_load_dword
 
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
-; GCN: {{buffer|flat}}_store_byte
 define amdgpu_kernel void @any_extend_vector_inreg_v16i8_to_v4i32(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(1) %arg1) local_unnamed_addr #0 {
+; GFX6-LABEL: any_extend_vector_inreg_v16i8_to_v4i32:
+; GFX6:       ; %bb.0: ; %bb
+; GFX6-NEXT:    s_load_dwordx4 s[12:15], s[4:5], 0x9
+; GFX6-NEXT:    s_mov_b32 s3, 0xf000
+; GFX6-NEXT:    s_mov_b32 s2, -1
+; GFX6-NEXT:    v_mov_b32_e32 v0, 0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_mov_b32 s0, s14
+; GFX6-NEXT:    s_mov_b32 s1, s15
+; GFX6-NEXT:    s_load_dwordx8 s[4:11], s[12:13], 0x0
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_load_dword s4, s[12:13], 0x8
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:13
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:15
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:14
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:8
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:11
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:10
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:4
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:6
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:1
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:3
+; GFX6-NEXT:    s_lshr_b32 s8, s9, 16
+; GFX6-NEXT:    s_waitcnt expcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v0, s6
+; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX6-NEXT:    s_lshl_b64 s[6:7], s[4:5], 8
+; GFX6-NEXT:    v_mov_b32_e32 v1, s11
+; GFX6-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:9
+; GFX6-NEXT:    s_waitcnt expcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s5
+; GFX6-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:2
+; GFX6-NEXT:    v_alignbit_b32 v0, s8, v0, 16
+; GFX6-NEXT:    s_waitcnt expcnt(0)
+; GFX6-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:12
+; GFX6-NEXT:    s_waitcnt expcnt(0)
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 8, v0
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
+; GFX6-NEXT:    buffer_store_byte v1, off, s[0:3], 0 offset:5
+; GFX6-NEXT:    buffer_store_byte v0, off, s[0:3], 0 offset:7
+; GFX6-NEXT:    s_endpgm
+;
+; GFX8-LABEL: any_extend_vector_inreg_v16i8_to_v4i32:
+; GFX8:       ; %bb.0: ; %bb
+; GFX8-NEXT:    s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_load_dword s0, s[8:9], 0x20
+; GFX8-NEXT:    s_lshr_b32 s6, s5, 24
+; GFX8-NEXT:    s_lshr_b32 s8, s2, 24
+; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX8-NEXT:    s_lshl_b64 s[2:3], s[0:1], 8
+; GFX8-NEXT:    s_add_u32 s4, s10, 13
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 15
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 14
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 8
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 11
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 10
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 4
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 6
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 1
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    v_mov_b32_e32 v0, s10
+; GFX8-NEXT:    v_mov_b32_e32 v1, s11
+; GFX8-NEXT:    s_add_u32 s4, s10, 3
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    s_add_u32 s4, s10, 9
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s7
+; GFX8-NEXT:    s_add_u32 s4, s10, 2
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s5, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    s_add_u32 s0, s10, 5
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s1, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s8
+; GFX8-NEXT:    s_add_u32 s0, s10, 12
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s1, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s3
+; GFX8-NEXT:    s_add_u32 s0, s10, 7
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_addc_u32 s1, s11, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s6
+; GFX8-NEXT:    flat_store_byte v[0:1], v2
+; GFX8-NEXT:    s_endpgm
 bb:
   %tmp2 = load <16 x i8>, ptr addrspace(1) %arg, align 16
   %tmp3 = extractelement <16 x i8> %tmp2, i64 4
-- 
cgit v1.2.3


From e5559ca45f211f2cdd9c81e46935afe1cc2e22ab Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 20 Jun 2025 06:44:14 -0500
Subject: [flang][OpenMP] Move lowering of ATOMIC to separate file, NFC
 (#144960)

---
 flang/lib/Lower/CMakeLists.txt    |   1 +
 flang/lib/Lower/OpenMP/Atomic.cpp | 510 ++++++++++++++++++++++++++++++++++++++
 flang/lib/Lower/OpenMP/Atomic.h   |  36 +++
 flang/lib/Lower/OpenMP/OpenMP.cpp | 470 +----------------------------------
 4 files changed, 549 insertions(+), 468 deletions(-)
 create mode 100644 flang/lib/Lower/OpenMP/Atomic.cpp
 create mode 100644 flang/lib/Lower/OpenMP/Atomic.h

diff --git a/flang/lib/Lower/CMakeLists.txt b/flang/lib/Lower/CMakeLists.txt
index 9c5db2b12651..8049cdf33317 100644
--- a/flang/lib/Lower/CMakeLists.txt
+++ b/flang/lib/Lower/CMakeLists.txt
@@ -23,6 +23,7 @@ add_flang_library(FortranLower
   LoweringOptions.cpp
   Mangler.cpp
   OpenACC.cpp
+  OpenMP/Atomic.cpp
   OpenMP/ClauseProcessor.cpp
   OpenMP/Clauses.cpp
   OpenMP/DataSharingProcessor.cpp
diff --git a/flang/lib/Lower/OpenMP/Atomic.cpp b/flang/lib/Lower/OpenMP/Atomic.cpp
new file mode 100644
index 000000000000..33a743f8f9dd
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Atomic.cpp
@@ -0,0 +1,510 @@
+//===-- Atomic.cpp -- Lowering of atomic constructs -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Atomic.h"
+#include "Clauses.h"
+#include "flang/Evaluate/expression.h"
+#include "flang/Evaluate/fold.h"
+#include "flang/Evaluate/tools.h"
+#include "flang/Lower/AbstractConverter.h"
+#include "flang/Lower/PFTBuilder.h"
+#include "flang/Lower/StatementContext.h"
+#include "flang/Lower/SymbolMap.h"
+#include "flang/Optimizer/Builder/FIRBuilder.h"
+#include "flang/Optimizer/Builder/Todo.h"
+#include "flang/Parser/parse-tree.h"
+#include "flang/Semantics/semantics.h"
+#include "flang/Semantics/type.h"
+#include "flang/Support/Fortran.h"
+#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <optional>
+#include <string>
+#include <type_traits>
+#include <variant>
+#include <vector>
+
+static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
+
+using namespace Fortran;
+
+// Don't import the entire Fortran::lower.
+namespace omp {
+using namespace Fortran::lower::omp;
+}
+
+[[maybe_unused]] static void
+dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
+  auto whatStr = [](int k) {
+    std::string txt = "?";
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
+    case parser::OpenMPAtomicConstruct::Analysis::None:
+      txt = "None";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Read:
+      txt = "Read";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Write:
+      txt = "Write";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::Update:
+      txt = "Update";
+      break;
+    }
+    switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
+    case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
+      txt += " | IfTrue";
+      break;
+    case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
+      txt += " | IfFalse";
+      break;
+    }
+    return txt;
+  };
+
+  auto exprStr = [&](const parser::TypedExpr &expr) {
+    if (auto *maybe = expr.get()) {
+      if (maybe->v)
+        return maybe->v->AsFortran();
+    }
+    return "<null>"s;
+  };
+  auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
+    if (auto *maybe = assign.get(); maybe && maybe->v) {
+      std::string str;
+      llvm::raw_string_ostream os(str);
+      maybe->v->AsFortran(os);
+      return str;
+    }
+    return "<null>"s;
+  };
+
+  const semantics::SomeExpr &atom = *analysis.atom.get()->v;
+
+  llvm::errs() << "Analysis {\n";
+  llvm::errs() << "  atom: " << atom.AsFortran() << "\n";
+  llvm::errs() << "  cond: " << exprStr(analysis.cond) << "\n";
+  llvm::errs() << "  op0 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op0.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op0.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "  op1 {\n";
+  llvm::errs() << "    what: " << whatStr(analysis.op1.what) << "\n";
+  llvm::errs() << "    assign: " << assignStr(analysis.op1.assign) << "\n";
+  llvm::errs() << "  }\n";
+  llvm::errs() << "}\n";
+}
+
+static bool isPointerAssignment(const evaluate::Assignment &assign) {
+  return common::visit(
+      common::visitors{
+          [](const evaluate::Assignment::BoundsSpec &) { return true; },
+          [](const evaluate::Assignment::BoundsRemapping &) { return true; },
+          [](const auto &) { return false; },
+      },
+      assign.u);
+}
+
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointBefore(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        mlir::Block::iterator(op));
+}
+
+static fir::FirOpBuilder::InsertPoint
+getInsertionPointAfter(mlir::Operation *op) {
+  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
+                                        ++mlir::Block::iterator(op));
+}
+
+static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
+                                       const omp::List<omp::Clause> &clauses) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  for (const omp::Clause &clause : clauses) {
+    if (clause.id != llvm::omp::Clause::OMPC_hint)
+      continue;
+    auto &hint = std::get<omp::clause::Hint>(clause.u);
+    auto maybeVal = evaluate::ToInt64(hint.v);
+    CHECK(maybeVal);
+    return builder.getI64IntegerAttr(*maybeVal);
+  }
+  return nullptr;
+}
+
+static mlir::omp::ClauseMemoryOrderKind
+getMemoryOrderKind(common::OmpMemoryOrderType kind) {
+  switch (kind) {
+  case common::OmpMemoryOrderType::Acq_Rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case common::OmpMemoryOrderType::Acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case common::OmpMemoryOrderType::Relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case common::OmpMemoryOrderType::Release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case common::OmpMemoryOrderType::Seq_Cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  }
+  llvm_unreachable("Unexpected kind");
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderKind(llvm::omp::Clause clauseId) {
+  switch (clauseId) {
+  case llvm::omp::Clause::OMPC_acq_rel:
+    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
+  case llvm::omp::Clause::OMPC_acquire:
+    return mlir::omp::ClauseMemoryOrderKind::Acquire;
+  case llvm::omp::Clause::OMPC_relaxed:
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  case llvm::omp::Clause::OMPC_release:
+    return mlir::omp::ClauseMemoryOrderKind::Release;
+  case llvm::omp::Clause::OMPC_seq_cst:
+    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
+  default:
+    return std::nullopt;
+  }
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getMemoryOrderFromRequires(const semantics::Scope &scope) {
+  // The REQUIRES construct is only allowed in the main program scope
+  // and module scope, but seems like we also accept it in a subprogram
+  // scope.
+  // For safety, traverse all enclosing scopes and check if their symbol
+  // contains REQUIRES.
+  for (const auto *sc{&scope}; sc->kind() != semantics::Scope::Kind::Global;
+       sc = &sc->parent()) {
+    const semantics::Symbol *sym = sc->symbol();
+    if (!sym)
+      continue;
+
+    const common::OmpMemoryOrderType *admo = common::visit(
+        [](auto &&s) {
+          using WithOmpDeclarative = semantics::WithOmpDeclarative;
+          if constexpr (std::is_convertible_v<decltype(s),
+                                              const WithOmpDeclarative &>) {
+            return s.ompAtomicDefaultMemOrder();
+          }
+          return static_cast<const common::OmpMemoryOrderType *>(nullptr);
+        },
+        sym->details());
+    if (admo)
+      return getMemoryOrderKind(*admo);
+  }
+
+  return std::nullopt;
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getDefaultAtomicMemOrder(semantics::SemanticsContext &semaCtx) {
+  unsigned version = semaCtx.langOptions().OpenMPVersion;
+  if (version > 50)
+    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
+  return std::nullopt;
+}
+
+static std::optional<mlir::omp::ClauseMemoryOrderKind>
+getAtomicMemoryOrder(semantics::SemanticsContext &semaCtx,
+                     const omp::List<omp::Clause> &clauses,
+                     const semantics::Scope &scope) {
+  for (const omp::Clause &clause : clauses) {
+    if (auto maybeKind = getMemoryOrderKind(clause.id))
+      return *maybeKind;
+  }
+
+  if (auto maybeKind = getMemoryOrderFromRequires(scope))
+    return *maybeKind;
+
+  return getDefaultAtomicMemOrder(semaCtx);
+}
+
+static mlir::omp::ClauseMemoryOrderKindAttr
+makeMemOrderAttr(lower::AbstractConverter &converter,
+                 std::optional<mlir::omp::ClauseMemoryOrderKind> maybeKind) {
+  if (maybeKind) {
+    return mlir::omp::ClauseMemoryOrderKindAttr::get(
+        converter.getFirOpBuilder().getContext(), *maybeKind);
+  }
+  return nullptr;
+}
+
+static mlir::Operation * //
+genAtomicRead(lower::AbstractConverter &converter,
+              semantics::SemanticsContext &semaCtx, mlir::Location loc,
+              lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+              const semantics::SomeExpr &atom,
+              const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+              std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+              fir::FirOpBuilder::InsertPoint preAt,
+              fir::FirOpBuilder::InsertPoint atomicAt,
+              fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  // If the atomic clause is read then the memory-order clause must
+  // not be release.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Release) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Acquire;
+    }
+  }
+
+  mlir::Value storeAddr =
+      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
+
+  mlir::Value toAddr = [&]() {
+    if (atomType == storeType)
+      return storeAddr;
+    return builder.createTemporary(loc, atomType, ".tmp.atomval");
+  }();
+
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
+      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
+      makeMemOrderAttr(converter, memOrder));
+
+  if (atomType != storeType) {
+    lower::ExprToValueMap overrides;
+    // The READ operation could be a part of UPDATE CAPTURE, so make sure
+    // we don't emit extra code into the body of the atomic op.
+    builder.restoreInsertionPoint(postAt);
+    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
+    overrides.try_emplace(&atom, load);
+
+    converter.overrideExprValues(&overrides);
+    mlir::Value value =
+        fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+    converter.resetExprOverrides();
+
+    builder.create<fir::StoreOp>(loc, value, storeAddr);
+  }
+  return op;
+}
+
+static mlir::Operation * //
+genAtomicWrite(lower::AbstractConverter &converter,
+               semantics::SemanticsContext &semaCtx, mlir::Location loc,
+               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+               const semantics::SomeExpr &atom,
+               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+               std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+               fir::FirOpBuilder::InsertPoint preAt,
+               fir::FirOpBuilder::InsertPoint atomicAt,
+               fir::FirOpBuilder::InsertPoint postAt) {
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  // If the atomic clause is write then the memory-order clause must
+  // not be acquire.
+  if (memOrder) {
+    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acquire) {
+      // Reset it back to the default.
+      memOrder = getDefaultAtomicMemOrder(semaCtx);
+    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
+      // The MLIR verifier doesn't like acq_rel either.
+      memOrder = mlir::omp::ClauseMemoryOrderKind::Release;
+    }
+  }
+
+  mlir::Value value =
+      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+  mlir::Value converted = builder.createConvert(loc, atomType, value);
+
+  builder.restoreInsertionPoint(atomicAt);
+  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
+      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
+  return op;
+}
+
+static mlir::Operation *
+genAtomicUpdate(lower::AbstractConverter &converter,
+                semantics::SemanticsContext &semaCtx, mlir::Location loc,
+                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
+                const semantics::SomeExpr &atom,
+                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+                fir::FirOpBuilder::InsertPoint preAt,
+                fir::FirOpBuilder::InsertPoint atomicAt,
+                fir::FirOpBuilder::InsertPoint postAt) {
+  lower::ExprToValueMap overrides;
+  lower::StatementContext naCtx;
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  builder.restoreInsertionPoint(preAt);
+
+  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
+
+  // This must exist by now.
+  semantics::SomeExpr input = *evaluate::GetConvertInput(assign.rhs);
+  std::vector<semantics::SomeExpr> args =
+      evaluate::GetTopLevelOperation(input).second;
+  assert(!args.empty() && "Update operation without arguments");
+  for (auto &arg : args) {
+    if (!evaluate::IsSameOrConvertOf(arg, atom)) {
+      mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
+      overrides.try_emplace(&arg, val);
+    }
+  }
+
+  builder.restoreInsertionPoint(atomicAt);
+  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
+      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
+
+  mlir::Region &region = updateOp->getRegion(0);
+  mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
+  mlir::Value localAtom = fir::getBase(block->getArgument(0));
+  overrides.try_emplace(&atom, localAtom);
+
+  converter.overrideExprValues(&overrides);
+  mlir::Value updated =
+      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
+  mlir::Value converted = builder.createConvert(loc, atomType, updated);
+  builder.create<mlir::omp::YieldOp>(loc, converted);
+  converter.resetExprOverrides();
+
+  builder.restoreInsertionPoint(postAt); // For naCtx cleanups
+  return updateOp;
+}
+
+static mlir::Operation *
+genAtomicOperation(lower::AbstractConverter &converter,
+                   semantics::SemanticsContext &semaCtx, mlir::Location loc,
+                   lower::StatementContext &stmtCtx, int action,
+                   mlir::Value atomAddr, const semantics::SomeExpr &atom,
+                   const evaluate::Assignment &assign, mlir::IntegerAttr hint,
+                   std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
+                   fir::FirOpBuilder::InsertPoint preAt,
+                   fir::FirOpBuilder::InsertPoint atomicAt,
+                   fir::FirOpBuilder::InsertPoint postAt) {
+  if (isPointerAssignment(assign)) {
+    TODO(loc, "Code generation for pointer assignment is not implemented yet");
+  }
+
+  // This function and the functions called here do not preserve the
+  // builder's insertion point, or set it to anything specific.
+  switch (action) {
+  case parser::OpenMPAtomicConstruct::Analysis::Read:
+    return genAtomicRead(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                         assign, hint, memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Write:
+    return genAtomicWrite(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                          assign, hint, memOrder, preAt, atomicAt, postAt);
+  case parser::OpenMPAtomicConstruct::Analysis::Update:
+    return genAtomicUpdate(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
+                           assign, hint, memOrder, preAt, atomicAt, postAt);
+  default:
+    return nullptr;
+  }
+}
+
+void Fortran::lower::omp::lowerAtomic(
+    AbstractConverter &converter, SymMap &symTable,
+    semantics::SemanticsContext &semaCtx, pft::Evaluation &eval,
+    const parser::OpenMPAtomicConstruct &construct) {
+  auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
+    if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
+      return &*maybe->v;
+    } else {
+      return nullptr;
+    }
+  };
+
+  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
+  auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
+  omp::List<omp::Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
+  lower::StatementContext stmtCtx;
+
+  const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
+  if (DumpAtomicAnalysis)
+    dumpAtomicAnalysis(analysis);
+
+  const semantics::SomeExpr &atom = *get(analysis.atom);
+  mlir::Location loc = converter.genLocation(construct.source);
+  mlir::Value atomAddr =
+      fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
+  mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
+  std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder =
+      getAtomicMemoryOrder(semaCtx, clauses,
+                           semaCtx.FindScope(construct.source));
+
+  if (auto *cond = get(analysis.cond)) {
+    (void)cond;
+    TODO(loc, "OpenMP ATOMIC COMPARE");
+  } else {
+    int action0 = analysis.op0.what & analysis.Action;
+    int action1 = analysis.op1.what & analysis.Action;
+    mlir::Operation *captureOp = nullptr;
+    fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
+    fir::FirOpBuilder::InsertPoint atomicAt, postAt;
+
+    if (construct.IsCapture()) {
+      // Capturing operation.
+      assert(action0 != analysis.None && action1 != analysis.None &&
+             "Expexcing two actions");
+      (void)action0;
+      (void)action1;
+      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
+          loc, hint, makeMemOrderAttr(converter, memOrder));
+      // Set the non-atomic insertion point to before the atomic.capture.
+      preAt = getInsertionPointBefore(captureOp);
+
+      mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
+      builder.setInsertionPointToEnd(block);
+      // Set the atomic insertion point to before the terminator inside
+      // atomic.capture.
+      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
+      atomicAt = getInsertionPointBefore(term);
+      postAt = getInsertionPointAfter(captureOp);
+      hint = nullptr;
+      memOrder = std::nullopt;
+    } else {
+      // Non-capturing operation.
+      assert(action0 != analysis.None && action1 == analysis.None &&
+             "Expexcing single action");
+      assert(!(analysis.op0.what & analysis.Condition));
+      postAt = atomicAt = preAt;
+    }
+
+    // The builder's insertion point needs to be specifically set before
+    // each call to `genAtomicOperation`.
+    mlir::Operation *firstOp = genAtomicOperation(
+        converter, semaCtx, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
+        *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
+    assert(firstOp && "Should have created an atomic operation");
+    atomicAt = getInsertionPointAfter(firstOp);
+
+    mlir::Operation *secondOp = nullptr;
+    if (analysis.op1.what != analysis.None) {
+      secondOp = genAtomicOperation(
+          converter, semaCtx, loc, stmtCtx, analysis.op1.what, atomAddr, atom,
+          *get(analysis.op1.assign), hint, memOrder, preAt, atomicAt, postAt);
+    }
+
+    if (construct.IsCapture()) {
+      // If this is a capture operation, the first/second ops will be inside
+      // of it. Set the insertion point to past the capture op itself.
+      builder.restoreInsertionPoint(postAt);
+    } else {
+      if (secondOp) {
+        builder.setInsertionPointAfter(secondOp);
+      } else {
+        builder.setInsertionPointAfter(firstOp);
+      }
+    }
+  }
+}
diff --git a/flang/lib/Lower/OpenMP/Atomic.h b/flang/lib/Lower/OpenMP/Atomic.h
new file mode 100644
index 000000000000..b83773b11300
--- /dev/null
+++ b/flang/lib/Lower/OpenMP/Atomic.h
@@ -0,0 +1,36 @@
+//===-- Atomic.h -- Lowering of atomic constructs -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+#ifndef FORTRAN_LOWER_OPENMP_ATOMIC_H
+#define FORTRAN_LOWER_OPENMP_ATOMIC_H
+
+namespace Fortran {
+namespace lower {
+class AbstractConverter;
+class SymMap;
+} // namespace lower
+
+namespace parser {
+struct OpenMPAtomicConstruct;
+}
+
+namespace pft {
+struct Evaluation;
+}
+
+namespace semantics {
+class SemanticsContext;
+}
+} // namespace Fortran
+
+namespace Fortran::lower::omp {
+void lowerAtomic(AbstractConverter &converter, SymMap &symTable,
+                 semantics::SemanticsContext &semaCtx, pft::Evaluation &eval,
+                 const parser::OpenMPAtomicConstruct &construct);
+}
+
+#endif // FORTRAN_LOWER_OPENMP_ATOMIC_H
diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index 1c51fd60d570..ddf58fd87444 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -12,6 +12,7 @@
 
 #include "flang/Lower/OpenMP.h"
 
+#include "Atomic.h"
 #include "ClauseProcessor.h"
 #include "Clauses.h"
 #include "DataSharingProcessor.h"
@@ -41,13 +42,10 @@
 #include "mlir/Transforms/RegionUtils.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
-#include "llvm/Support/CommandLine.h"
 
 using namespace Fortran::lower::omp;
 using namespace Fortran::common::openmp;
 
-static llvm::cl::opt<bool> DumpAtomicAnalysis("fdebug-dump-atomic-analysis");
-
 //===----------------------------------------------------------------------===//
 // Code generation helper functions
 //===----------------------------------------------------------------------===//
@@ -1127,16 +1125,6 @@ markDeclareTarget(mlir::Operation *op, lower::AbstractConverter &converter,
   declareTargetOp.setDeclareTarget(deviceType, captureClause);
 }
 
-static bool isPointerAssignment(const evaluate::Assignment &assign) {
-  return common::visit(
-      common::visitors{
-          [](const evaluate::Assignment::BoundsSpec &) { return true; },
-          [](const evaluate::Assignment::BoundsRemapping &) { return true; },
-          [](const auto &) { return false; },
-      },
-      assign.u);
-}
-
 //===----------------------------------------------------------------------===//
 // Op body generation helper structures and functions
 //===----------------------------------------------------------------------===//
@@ -2694,308 +2682,6 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
       queue, item, clauseOps);
 }
 
-//===----------------------------------------------------------------------===//
-// Code generation for atomic operations
-//===----------------------------------------------------------------------===//
-static fir::FirOpBuilder::InsertPoint
-getInsertionPointBefore(mlir::Operation *op) {
-  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
-                                        mlir::Block::iterator(op));
-}
-
-static fir::FirOpBuilder::InsertPoint
-getInsertionPointAfter(mlir::Operation *op) {
-  return fir::FirOpBuilder::InsertPoint(op->getBlock(),
-                                        ++mlir::Block::iterator(op));
-}
-
-static mlir::IntegerAttr getAtomicHint(lower::AbstractConverter &converter,
-                                       const List<Clause> &clauses) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  for (const Clause &clause : clauses) {
-    if (clause.id != llvm::omp::Clause::OMPC_hint)
-      continue;
-    auto &hint = std::get<clause::Hint>(clause.u);
-    auto maybeVal = evaluate::ToInt64(hint.v);
-    CHECK(maybeVal);
-    return builder.getI64IntegerAttr(*maybeVal);
-  }
-  return nullptr;
-}
-
-static mlir::omp::ClauseMemoryOrderKind
-getMemoryOrderKind(common::OmpMemoryOrderType kind) {
-  switch (kind) {
-  case common::OmpMemoryOrderType::Acq_Rel:
-    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-  case common::OmpMemoryOrderType::Acquire:
-    return mlir::omp::ClauseMemoryOrderKind::Acquire;
-  case common::OmpMemoryOrderType::Relaxed:
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  case common::OmpMemoryOrderType::Release:
-    return mlir::omp::ClauseMemoryOrderKind::Release;
-  case common::OmpMemoryOrderType::Seq_Cst:
-    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-  }
-  llvm_unreachable("Unexpected kind");
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getMemoryOrderKind(llvm::omp::Clause clauseId) {
-  switch (clauseId) {
-  case llvm::omp::Clause::OMPC_acq_rel:
-    return mlir::omp::ClauseMemoryOrderKind::Acq_rel;
-  case llvm::omp::Clause::OMPC_acquire:
-    return mlir::omp::ClauseMemoryOrderKind::Acquire;
-  case llvm::omp::Clause::OMPC_relaxed:
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  case llvm::omp::Clause::OMPC_release:
-    return mlir::omp::ClauseMemoryOrderKind::Release;
-  case llvm::omp::Clause::OMPC_seq_cst:
-    return mlir::omp::ClauseMemoryOrderKind::Seq_cst;
-  default:
-    return std::nullopt;
-  }
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getMemoryOrderFromRequires(const semantics::Scope &scope) {
-  // The REQUIRES construct is only allowed in the main program scope
-  // and module scope, but seems like we also accept it in a subprogram
-  // scope.
-  // For safety, traverse all enclosing scopes and check if their symbol
-  // contains REQUIRES.
-  for (const auto *sc{&scope}; sc->kind() != semantics::Scope::Kind::Global;
-       sc = &sc->parent()) {
-    const semantics::Symbol *sym = sc->symbol();
-    if (!sym)
-      continue;
-
-    const common::OmpMemoryOrderType *admo = common::visit(
-        [](auto &&s) {
-          using WithOmpDeclarative = semantics::WithOmpDeclarative;
-          if constexpr (std::is_convertible_v<decltype(s),
-                                              const WithOmpDeclarative &>) {
-            return s.ompAtomicDefaultMemOrder();
-          }
-          return static_cast<const common::OmpMemoryOrderType *>(nullptr);
-        },
-        sym->details());
-    if (admo)
-      return getMemoryOrderKind(*admo);
-  }
-
-  return std::nullopt;
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getDefaultAtomicMemOrder(semantics::SemanticsContext &semaCtx) {
-  unsigned version = semaCtx.langOptions().OpenMPVersion;
-  if (version > 50)
-    return mlir::omp::ClauseMemoryOrderKind::Relaxed;
-  return std::nullopt;
-}
-
-static std::optional<mlir::omp::ClauseMemoryOrderKind>
-getAtomicMemoryOrder(semantics::SemanticsContext &semaCtx,
-                     const List<Clause> &clauses,
-                     const semantics::Scope &scope) {
-  for (const Clause &clause : clauses) {
-    if (auto maybeKind = getMemoryOrderKind(clause.id))
-      return *maybeKind;
-  }
-
-  if (auto maybeKind = getMemoryOrderFromRequires(scope))
-    return *maybeKind;
-
-  return getDefaultAtomicMemOrder(semaCtx);
-}
-
-static mlir::omp::ClauseMemoryOrderKindAttr
-makeMemOrderAttr(lower::AbstractConverter &converter,
-                 std::optional<mlir::omp::ClauseMemoryOrderKind> maybeKind) {
-  if (maybeKind) {
-    return mlir::omp::ClauseMemoryOrderKindAttr::get(
-        converter.getFirOpBuilder().getContext(), *maybeKind);
-  }
-  return nullptr;
-}
-
-static mlir::Operation * //
-genAtomicRead(lower::AbstractConverter &converter,
-              semantics::SemanticsContext &semaCtx, mlir::Location loc,
-              lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-              const semantics::SomeExpr &atom,
-              const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-              std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-              fir::FirOpBuilder::InsertPoint preAt,
-              fir::FirOpBuilder::InsertPoint atomicAt,
-              fir::FirOpBuilder::InsertPoint postAt) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  // If the atomic clause is read then the memory-order clause must
-  // not be release.
-  if (memOrder) {
-    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Release) {
-      // Reset it back to the default.
-      memOrder = getDefaultAtomicMemOrder(semaCtx);
-    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
-      // The MLIR verifier doesn't like acq_rel either.
-      memOrder = mlir::omp::ClauseMemoryOrderKind::Acquire;
-    }
-  }
-
-  mlir::Value storeAddr =
-      fir::getBase(converter.genExprAddr(assign.lhs, stmtCtx, &loc));
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-  mlir::Type storeType = fir::unwrapRefType(storeAddr.getType());
-
-  mlir::Value toAddr = [&]() {
-    if (atomType == storeType)
-      return storeAddr;
-    return builder.createTemporary(loc, atomType, ".tmp.atomval");
-  }();
-
-  builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicReadOp>(
-      loc, atomAddr, toAddr, mlir::TypeAttr::get(atomType), hint,
-      makeMemOrderAttr(converter, memOrder));
-
-  if (atomType != storeType) {
-    lower::ExprToValueMap overrides;
-    // The READ operation could be a part of UPDATE CAPTURE, so make sure
-    // we don't emit extra code into the body of the atomic op.
-    builder.restoreInsertionPoint(postAt);
-    mlir::Value load = builder.create<fir::LoadOp>(loc, toAddr);
-    overrides.try_emplace(&atom, load);
-
-    converter.overrideExprValues(&overrides);
-    mlir::Value value =
-        fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-    converter.resetExprOverrides();
-
-    builder.create<fir::StoreOp>(loc, value, storeAddr);
-  }
-  return op;
-}
-
-static mlir::Operation * //
-genAtomicWrite(lower::AbstractConverter &converter,
-               semantics::SemanticsContext &semaCtx, mlir::Location loc,
-               lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-               const semantics::SomeExpr &atom,
-               const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-               std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-               fir::FirOpBuilder::InsertPoint preAt,
-               fir::FirOpBuilder::InsertPoint atomicAt,
-               fir::FirOpBuilder::InsertPoint postAt) {
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  // If the atomic clause is write then the memory-order clause must
-  // not be acquire.
-  if (memOrder) {
-    if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acquire) {
-      // Reset it back to the default.
-      memOrder = getDefaultAtomicMemOrder(semaCtx);
-    } else if (*memOrder == mlir::omp::ClauseMemoryOrderKind::Acq_rel) {
-      // The MLIR verifier doesn't like acq_rel either.
-      memOrder = mlir::omp::ClauseMemoryOrderKind::Release;
-    }
-  }
-
-  mlir::Value value =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-  mlir::Value converted = builder.createConvert(loc, atomType, value);
-
-  builder.restoreInsertionPoint(atomicAt);
-  mlir::Operation *op = builder.create<mlir::omp::AtomicWriteOp>(
-      loc, atomAddr, converted, hint, makeMemOrderAttr(converter, memOrder));
-  return op;
-}
-
-static mlir::Operation *
-genAtomicUpdate(lower::AbstractConverter &converter,
-                semantics::SemanticsContext &semaCtx, mlir::Location loc,
-                lower::StatementContext &stmtCtx, mlir::Value atomAddr,
-                const semantics::SomeExpr &atom,
-                const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-                fir::FirOpBuilder::InsertPoint preAt,
-                fir::FirOpBuilder::InsertPoint atomicAt,
-                fir::FirOpBuilder::InsertPoint postAt) {
-  lower::ExprToValueMap overrides;
-  lower::StatementContext naCtx;
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  builder.restoreInsertionPoint(preAt);
-
-  mlir::Type atomType = fir::unwrapRefType(atomAddr.getType());
-
-  // This must exist by now.
-  SomeExpr input = *Fortran::evaluate::GetConvertInput(assign.rhs);
-  std::vector<SomeExpr> args{
-      Fortran::evaluate::GetTopLevelOperation(input).second};
-  assert(!args.empty() && "Update operation without arguments");
-  for (auto &arg : args) {
-    if (!Fortran::evaluate::IsSameOrConvertOf(arg, atom)) {
-      mlir::Value val = fir::getBase(converter.genExprValue(arg, naCtx, &loc));
-      overrides.try_emplace(&arg, val);
-    }
-  }
-
-  builder.restoreInsertionPoint(atomicAt);
-  auto updateOp = builder.create<mlir::omp::AtomicUpdateOp>(
-      loc, atomAddr, hint, makeMemOrderAttr(converter, memOrder));
-
-  mlir::Region &region = updateOp->getRegion(0);
-  mlir::Block *block = builder.createBlock(&region, {}, {atomType}, {loc});
-  mlir::Value localAtom = fir::getBase(block->getArgument(0));
-  overrides.try_emplace(&atom, localAtom);
-
-  converter.overrideExprValues(&overrides);
-  mlir::Value updated =
-      fir::getBase(converter.genExprValue(assign.rhs, stmtCtx, &loc));
-  mlir::Value converted = builder.createConvert(loc, atomType, updated);
-  builder.create<mlir::omp::YieldOp>(loc, converted);
-  converter.resetExprOverrides();
-
-  builder.restoreInsertionPoint(postAt); // For naCtx cleanups
-  return updateOp;
-}
-
-static mlir::Operation *
-genAtomicOperation(lower::AbstractConverter &converter,
-                   semantics::SemanticsContext &semaCtx, mlir::Location loc,
-                   lower::StatementContext &stmtCtx, int action,
-                   mlir::Value atomAddr, const semantics::SomeExpr &atom,
-                   const evaluate::Assignment &assign, mlir::IntegerAttr hint,
-                   std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder,
-                   fir::FirOpBuilder::InsertPoint preAt,
-                   fir::FirOpBuilder::InsertPoint atomicAt,
-                   fir::FirOpBuilder::InsertPoint postAt) {
-  if (isPointerAssignment(assign)) {
-    TODO(loc, "Code generation for pointer assignment is not implemented yet");
-  }
-
-  // This function and the functions called here do not preserve the
-  // builder's insertion point, or set it to anything specific.
-  switch (action) {
-  case parser::OpenMPAtomicConstruct::Analysis::Read:
-    return genAtomicRead(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                         assign, hint, memOrder, preAt, atomicAt, postAt);
-  case parser::OpenMPAtomicConstruct::Analysis::Write:
-    return genAtomicWrite(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                          assign, hint, memOrder, preAt, atomicAt, postAt);
-  case parser::OpenMPAtomicConstruct::Analysis::Update:
-    return genAtomicUpdate(converter, semaCtx, loc, stmtCtx, atomAddr, atom,
-                           assign, hint, memOrder, preAt, atomicAt, postAt);
-  default:
-    return nullptr;
-  }
-}
-
 //===----------------------------------------------------------------------===//
 // Code generation functions for the standalone version of constructs that can
 // also be a leaf of a composite construct
@@ -3900,163 +3586,11 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
 // OpenMPConstruct visitors
 //===----------------------------------------------------------------------===//
 
-[[maybe_unused]] static void
-dumpAtomicAnalysis(const parser::OpenMPAtomicConstruct::Analysis &analysis) {
-  auto whatStr = [](int k) {
-    std::string txt = "?";
-    switch (k & parser::OpenMPAtomicConstruct::Analysis::Action) {
-    case parser::OpenMPAtomicConstruct::Analysis::None:
-      txt = "None";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Read:
-      txt = "Read";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Write:
-      txt = "Write";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::Update:
-      txt = "Update";
-      break;
-    }
-    switch (k & parser::OpenMPAtomicConstruct::Analysis::Condition) {
-    case parser::OpenMPAtomicConstruct::Analysis::IfTrue:
-      txt += " | IfTrue";
-      break;
-    case parser::OpenMPAtomicConstruct::Analysis::IfFalse:
-      txt += " | IfFalse";
-      break;
-    }
-    return txt;
-  };
-
-  auto exprStr = [&](const parser::TypedExpr &expr) {
-    if (auto *maybe = expr.get()) {
-      if (maybe->v)
-        return maybe->v->AsFortran();
-    }
-    return "<null>"s;
-  };
-  auto assignStr = [&](const parser::AssignmentStmt::TypedAssignment &assign) {
-    if (auto *maybe = assign.get(); maybe && maybe->v) {
-      std::string str;
-      llvm::raw_string_ostream os(str);
-      maybe->v->AsFortran(os);
-      return str;
-    }
-    return "<null>"s;
-  };
-
-  const SomeExpr &atom = *analysis.atom.get()->v;
-
-  llvm::errs() << "Analysis {\n";
-  llvm::errs() << "  atom: " << atom.AsFortran() << "\n";
-  llvm::errs() << "  cond: " << exprStr(analysis.cond) << "\n";
-  llvm::errs() << "  op0 {\n";
-  llvm::errs() << "    what: " << whatStr(analysis.op0.what) << "\n";
-  llvm::errs() << "    assign: " << assignStr(analysis.op0.assign) << "\n";
-  llvm::errs() << "  }\n";
-  llvm::errs() << "  op1 {\n";
-  llvm::errs() << "    what: " << whatStr(analysis.op1.what) << "\n";
-  llvm::errs() << "    assign: " << assignStr(analysis.op1.assign) << "\n";
-  llvm::errs() << "  }\n";
-  llvm::errs() << "}\n";
-}
-
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
                    semantics::SemanticsContext &semaCtx,
                    lower::pft::Evaluation &eval,
                    const parser::OpenMPAtomicConstruct &construct) {
-  auto get = [](auto &&typedWrapper) -> decltype(&*typedWrapper.get()->v) {
-    if (auto *maybe = typedWrapper.get(); maybe && maybe->v) {
-      return &*maybe->v;
-    } else {
-      return nullptr;
-    }
-  };
-
-  fir::FirOpBuilder &builder = converter.getFirOpBuilder();
-  auto &dirSpec = std::get<parser::OmpDirectiveSpecification>(construct.t);
-  List<Clause> clauses = makeClauses(dirSpec.Clauses(), semaCtx);
-  lower::StatementContext stmtCtx;
-
-  const parser::OpenMPAtomicConstruct::Analysis &analysis = construct.analysis;
-  if (DumpAtomicAnalysis)
-    dumpAtomicAnalysis(analysis);
-
-  const semantics::SomeExpr &atom = *get(analysis.atom);
-  mlir::Location loc = converter.genLocation(construct.source);
-  mlir::Value atomAddr =
-      fir::getBase(converter.genExprAddr(atom, stmtCtx, &loc));
-  mlir::IntegerAttr hint = getAtomicHint(converter, clauses);
-  std::optional<mlir::omp::ClauseMemoryOrderKind> memOrder =
-      getAtomicMemoryOrder(semaCtx, clauses,
-                           semaCtx.FindScope(construct.source));
-
-  if (auto *cond = get(analysis.cond)) {
-    (void)cond;
-    TODO(loc, "OpenMP ATOMIC COMPARE");
-  } else {
-    int action0 = analysis.op0.what & analysis.Action;
-    int action1 = analysis.op1.what & analysis.Action;
-    mlir::Operation *captureOp = nullptr;
-    fir::FirOpBuilder::InsertPoint preAt = builder.saveInsertionPoint();
-    fir::FirOpBuilder::InsertPoint atomicAt, postAt;
-
-    if (construct.IsCapture()) {
-      // Capturing operation.
-      assert(action0 != analysis.None && action1 != analysis.None &&
-             "Expexcing two actions");
-      (void)action0;
-      (void)action1;
-      captureOp = builder.create<mlir::omp::AtomicCaptureOp>(
-          loc, hint, makeMemOrderAttr(converter, memOrder));
-      // Set the non-atomic insertion point to before the atomic.capture.
-      preAt = getInsertionPointBefore(captureOp);
-
-      mlir::Block *block = builder.createBlock(&captureOp->getRegion(0));
-      builder.setInsertionPointToEnd(block);
-      // Set the atomic insertion point to before the terminator inside
-      // atomic.capture.
-      mlir::Operation *term = builder.create<mlir::omp::TerminatorOp>(loc);
-      atomicAt = getInsertionPointBefore(term);
-      postAt = getInsertionPointAfter(captureOp);
-      hint = nullptr;
-      memOrder = std::nullopt;
-    } else {
-      // Non-capturing operation.
-      assert(action0 != analysis.None && action1 == analysis.None &&
-             "Expexcing single action");
-      assert(!(analysis.op0.what & analysis.Condition));
-      postAt = atomicAt = preAt;
-    }
-
-    // The builder's insertion point needs to be specifically set before
-    // each call to `genAtomicOperation`.
-    mlir::Operation *firstOp = genAtomicOperation(
-        converter, semaCtx, loc, stmtCtx, analysis.op0.what, atomAddr, atom,
-        *get(analysis.op0.assign), hint, memOrder, preAt, atomicAt, postAt);
-    assert(firstOp && "Should have created an atomic operation");
-    atomicAt = getInsertionPointAfter(firstOp);
-
-    mlir::Operation *secondOp = nullptr;
-    if (analysis.op1.what != analysis.None) {
-      secondOp = genAtomicOperation(
-          converter, semaCtx, loc, stmtCtx, analysis.op1.what, atomAddr, atom,
-          *get(analysis.op1.assign), hint, memOrder, preAt, atomicAt, postAt);
-    }
-
-    if (construct.IsCapture()) {
-      // If this is a capture operation, the first/second ops will be inside
-      // of it. Set the insertion point to past the capture op itself.
-      builder.restoreInsertionPoint(postAt);
-    } else {
-      if (secondOp) {
-        builder.setInsertionPointAfter(secondOp);
-      } else {
-        builder.setInsertionPointAfter(firstOp);
-      }
-    }
-  }
+  lowerAtomic(converter, symTable, semaCtx, eval, construct);
 }
 
 static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable,
-- 
cgit v1.2.3


From 1b83f10072b322a206ffcaf737b42fe5c2d95b89 Mon Sep 17 00:00:00 2001
From: Vigneshwar Jayakumar <vigneshwar.jayakumar@amd.com>
Date: Fri, 20 Jun 2025 07:44:23 -0400
Subject: [AMDGPU] Fix to prevent sinking of PERMLANE_SWAP instruction
 (#144423)

Permlane_swap instruction depends on exec mask, added isConvergent flag
to prevent sinking of instruction.

Fixes: SWDEV-537232
---
 llvm/lib/Target/AMDGPU/VOP1Instructions.td         |  6 +-
 llvm/lib/Target/AMDGPU/VOPInstructions.td          |  1 +
 .../AMDGPU/machine-sink-ignorable-exec-use.mir     | 67 ++++++++++++++++++++++
 3 files changed, 73 insertions(+), 1 deletion(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
index 926df955881e..02b912bcfb9e 100644
--- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td
@@ -774,7 +774,8 @@ defm V_PRNG_B32 : VOP1Inst <"v_prng_b32", VOP_I32_I32, int_amdgcn_prng_b32>;
 
 let Constraints = "$vdst = $vdst_in, $src0_out = $src0",
      DisableEncoding="$vdst_in,$src0_out",
-     SchedRW = [Write32Bit, Write32Bit] in {
+     SchedRW = [Write32Bit, Write32Bit],
+     isConvergent = 1 in {
 let SubtargetPredicate = HasPermlane16Swap in {
 defm V_PERMLANE16_SWAP_B32 : VOP1Inst<"v_permlane16_swap_b32", VOP_PERMLANE_SWAP>;
 }
@@ -1549,8 +1550,11 @@ defm V_CVT_PK_F32_FP8    : VOP1_Real_NoDstSel_SDWA_gfx9<0x56>;
 defm V_CVT_PK_F32_BF8    : VOP1_Real_NoDstSel_SDWA_gfx9<0x57>;
 
 defm V_PRNG_B32            : VOP1_Real_gfx9 <0x58>;
+
+let isConvergent = 1 in {
 defm V_PERMLANE16_SWAP_B32 : VOP1_OpSel_Real_e32e64_gfx9<0x059>;
 defm V_PERMLANE32_SWAP_B32 : VOP1_OpSel_Real_e32e64_gfx9<0x05a>;
+}
 
 class MovDPP8Pattern<Predicate Pred, Instruction Inst, ValueType vt> : GCNPat <
   (vt (int_amdgcn_mov_dpp8 vt:$src, timm:$dpp8)),
diff --git a/llvm/lib/Target/AMDGPU/VOPInstructions.td b/llvm/lib/Target/AMDGPU/VOPInstructions.td
index 6045f59d1f04..19d490465f16 100644
--- a/llvm/lib/Target/AMDGPU/VOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOPInstructions.td
@@ -15,6 +15,7 @@ class LetDummies {
   bit isConvertibleToThreeAddress;
   bit isMoveImm;
   bit isReMaterializable;
+  bit isConvergent;
   bit isAsCheapAsAMove;
   bit FPDPRounding;
   Predicate SubtargetPredicate;
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-ignorable-exec-use.mir b/llvm/test/CodeGen/AMDGPU/machine-sink-ignorable-exec-use.mir
index 0fc31ea9d643..ed22b353b066 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-ignorable-exec-use.mir
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-ignorable-exec-use.mir
@@ -733,3 +733,70 @@ body:             |
     liveins: $vgpr0, $vgpr1, $vgpr2_vgpr3, $vcc
     S_ENDPGM 0
 ...
+---
+name: test_no_sink_permlane_swap
+tracksRegLiveness: true
+machineFunctionInfo:
+  isEntryFunction: true
+body: |
+  ; GFX9-LABEL: name: test_no_sink_permlane_swap
+  ; GFX9: bb.0:
+  ; GFX9-NEXT:   successors: %bb.2(0x40000000), %bb.1(0x40000000)
+  ; GFX9-NEXT:   liveins: $vgpr0
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+  ; GFX9-NEXT:   [[S_MOV_B64_:%[0-9]+]]:sreg_64 = S_MOV_B64 0
+  ; GFX9-NEXT:   [[COPY1:%[0-9]+]]:vreg_64 = COPY [[S_MOV_B64_]]
+  ; GFX9-NEXT:   [[GLOBAL_LOAD_DWORD:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_DWORD killed [[COPY1]], 0, 0, implicit $exec :: (load (s32), addrspace 1)
+  ; GFX9-NEXT:   [[V_PERMLANE32_SWAP_B32_e64_:%[0-9]+]]:vgpr_32, [[V_PERMLANE32_SWAP_B32_e64_1:%[0-9]+]]:vgpr_32 = V_PERMLANE32_SWAP_B32_e64 [[GLOBAL_LOAD_DWORD]], [[GLOBAL_LOAD_DWORD]], 0, 0, implicit $exec
+  ; GFX9-NEXT:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GFX9-NEXT:   [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1
+  ; GFX9-NEXT:   [[V_CMP_LT_I32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_I32_e64 [[COPY2]], [[S_MOV_B32_]], implicit $exec
+  ; GFX9-NEXT:   [[SI_IF:%[0-9]+]]:sreg_64 = SI_IF [[V_CMP_LT_I32_e64_]], %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; GFX9-NEXT:   S_BRANCH %bb.1
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT: bb.1:
+  ; GFX9-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[V_MAX_I32_e64_:%[0-9]+]]:vgpr_32 = V_MAX_I32_e64 [[V_PERMLANE32_SWAP_B32_e64_]], [[V_PERMLANE32_SWAP_B32_e64_1]], implicit $exec
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT: bb.2:
+  ; GFX9-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT:   [[PHI:%[0-9]+]]:vgpr_32 = PHI [[V_MOV_B32_e32_]], %bb.0, [[V_MAX_I32_e64_]], %bb.1
+  ; GFX9-NEXT:   SI_END_CF [[SI_IF]], implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+  ; GFX9-NEXT: {{  $}}
+  ; GFX9-NEXT: bb.3:
+  ; GFX9-NEXT:   S_ENDPGM 0, implicit [[PHI]]
+  bb.0:
+    successors: %bb.2(0x40000000), %bb.1(0x40000000)
+    liveins: $vgpr0
+
+    %0:vgpr_32 = COPY $vgpr0
+    %1:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+    %2:sreg_64 = S_MOV_B64 0
+    %3:vreg_64 = COPY %2
+    %4:vgpr_32 = GLOBAL_LOAD_DWORD killed %3, 0, 0, implicit $exec :: (load (s32), addrspace 1)
+    %5:vgpr_32, %6:vgpr_32 = V_PERMLANE32_SWAP_B32_e64 %4, %4, 0, 0, implicit $exec
+    %7:vgpr_32 = COPY $vgpr0
+    %8:sreg_32 = S_MOV_B32 1
+    %9:sreg_64 = V_CMP_LT_I32_e64 %7, %8, implicit $exec
+    %10:sreg_64 = COPY %9
+    %11:sreg_64 = SI_IF %10, %bb.2, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+    S_BRANCH %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %12:vgpr_32 = V_MAX_I32_e64 %5, %6, implicit $exec
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+    %13:vgpr_32 = PHI %1, %bb.0, %12, %bb.1
+    SI_END_CF %11, implicit-def dead $exec, implicit-def dead $scc, implicit $exec
+
+  bb.3:
+    S_ENDPGM 0, implicit %13
+...
-- 
cgit v1.2.3


From e8be733a3c3347207c162fc83e8dbe02dad2a952 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 20 Jun 2025 12:45:20 +0100
Subject: [VPlan] Remove redundant ExtractLastElement from vector-to-scalar
 VPI.

Recipes that are vector-to-scalar are guaranteed to generate a scalar
value, so the extract is redundant after VPlan unrolling. Remove it.

This removes unneeded ExtractLastElement VPInstruction of reduction
result computations.
---
 llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h            | 3 +++
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp            | 8 ++++++++
 llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll | 3 +--
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
index b2535fe3aa57..ae91e310f759 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
+++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
@@ -192,6 +192,9 @@ inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
 /// Match a VPValue, capturing it if we match.
 inline bind_ty<VPValue> m_VPValue(VPValue *&V) { return V; }
 
+/// Match a VPInstruction, capturing if we match.
+inline bind_ty<VPInstruction> m_VPInstruction(VPInstruction *&V) { return V; }
+
 template <typename Ops_t, unsigned Opcode, bool Commutative,
           typename... RecipeTys>
 struct Recipe_match {
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 60a8837fb76a..b6ca50549fa3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1163,6 +1163,14 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     });
     return;
   }
+
+  VPInstruction *OpVPI;
+  if (match(Def, m_VPInstruction<VPInstruction::ExtractLastElement>(
+                     m_VPInstruction(OpVPI))) &&
+      OpVPI->isVectorToScalar()) {
+    Def->replaceAllUsesWith(OpVPI);
+    return;
+  }
 }
 
 void VPlanTransforms::simplifyRecipes(VPlan &Plan, Type &CanonicalIVTy) {
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
index 8095f258ea18..307ab3a87e9d 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vplan-printing.ll
@@ -106,13 +106,12 @@ define i32 @print_partial_reduction(ptr %a, ptr %b) {
 ; CHECK-EMPTY:
 ; CHECK-NEXT: middle.block:
 ; CHECK-NEXT:   EMIT vp<[[RED_RESULT:%.+]]> = compute-reduction-result ir<%accum>, ir<%add>
-; CHECK-NEXT:   EMIT vp<[[EXTRACT:%.+]]> = extract-last-element vp<[[RED_RESULT]]>
 ; CHECK-NEXT:   EMIT vp<[[CMP:%.+]]> = icmp eq ir<1024>, ir<1024>
 ; CHECK-NEXT:   EMIT branch-on-cond vp<[[CMP]]>
 ; CHECK-NEXT: Successor(s): ir-bb<exit>, ir-bb<scalar.ph>
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<exit>:
-; CHECK-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[EXTRACT]]> from middle.block)
+; CHECK-NEXT:   IR   %add.lcssa = phi i32 [ %add, %for.body ] (extra operand: vp<[[RED_RESULT]]> from middle.block)
 ; CHECK-NEXT: No successors
 ; CHECK-EMPTY:
 ; CHECK-NEXT: ir-bb<scalar.ph>:
-- 
cgit v1.2.3


From 8650c801381d5640018a2fab32932ee41ad27797 Mon Sep 17 00:00:00 2001
From: NimishMishra <42909663+NimishMishra@users.noreply.github.com>
Date: Fri, 20 Jun 2025 17:22:40 +0530
Subject: [flang][OpenMP] Do not skip privatization of linear variable if it is
 OmpPreDetermined (#144315)

Current implementation of linear clause skips privatisation of all
linear variables during the FIR generation phase, since linear variables
are handled in their entirety by the OpenMP IRBuilder. However,
"implicit" linear variables (like OmpPreDetermined) cannot be skipped,
since FIR generation requires privatized symbols. This patch adds checks
to skip the same.


Fixes https://github.com/llvm/llvm-project/issues/142935
---
 flang/lib/Lower/OpenMP/DataSharingProcessor.cpp | 40 +++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
index 1b8670b379f8..3fae3f3a0ddf 100644
--- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
+++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp
@@ -204,6 +204,42 @@ void DataSharingProcessor::collectOmpObjectListSymbol(
 }
 
 void DataSharingProcessor::collectSymbolsForPrivatization() {
+  // Add checks here for exceptional cases where privatization is not
+  // needed and be deferred to a later phase (like OpenMP IRBuilder).
+  // Such cases are suggested to be clearly documented and explained
+  // instead of being silently skipped
+  auto isException = [&](const Fortran::semantics::Symbol *sym) -> bool {
+    // `OmpPreDetermined` symbols cannot be exceptions since
+    // their privatized symbols are heavily used in FIR.
+    if (sym->test(Fortran::semantics::Symbol::Flag::OmpPreDetermined))
+      return false;
+
+    // The handling of linear clause is deferred to the OpenMP
+    // IRBuilder which is responsible for all its aspects,
+    // including privatization. Privatizing linear variables at this point would
+    // cause the following structure:
+    //
+    // omp.op linear(%linear = %step : !fir.ref<type>) {
+    //	Use %linear in this BB
+    // }
+    //
+    // to be changed to the following:
+    //
+    // omp. op linear(%linear = %step : !fir.ref<type>)
+    // 	private(%linear -> %arg0 : !fir.ref<i32>) {
+    //	Declare and use %arg0 in this BB
+    // }
+    //
+    // The OpenMP IRBuilder needs to map the linear MLIR value
+    // (i.e. %linear) to its `uses` in the BB to correctly
+    // implement the functionalities of linear clause. However,
+    // privatizing here disallows the IRBuilder to
+    // draw a relation between %linear and %arg0. Hence skip.
+    if (sym->test(Fortran::semantics::Symbol::Flag::OmpLinear))
+      return true;
+    return false;
+  };
+
   for (const omp::Clause &clause : clauses) {
     if (const auto &privateClause =
             std::get_if<omp::clause::Private>(&clause.u)) {
@@ -222,10 +258,10 @@ void DataSharingProcessor::collectSymbolsForPrivatization() {
   }
 
   // TODO For common blocks, add the underlying objects within the block. Doing
-  // so, we won't need to explicitely handle block objects (or forget to do
+  // so, we won't need to explicitly handle block objects (or forget to do
   // so).
   for (auto *sym : explicitlyPrivatizedSymbols)
-    if (!sym->test(Fortran::semantics::Symbol::Flag::OmpLinear))
+    if (!isException(sym))
       allPrivatizedSymbols.insert(sym);
 }
 
-- 
cgit v1.2.3


From e970f59e6b20dddc4369735affb79ca9be240c1c Mon Sep 17 00:00:00 2001
From: NimishMishra <42909663+NimishMishra@users.noreply.github.com>
Date: Fri, 20 Jun 2025 17:23:00 +0530
Subject: [flang][OpenMP] Reintroduce TODO for FIR lowering of linear clause
 (#144883)

Current design of the linear clause lowering and translation shifts all
responsibility for handling the clause (like privatisation, linear
stepping, finalisation, and emission of synchronisation barriers) to the
IRBuilder. However in certain corner cases (like associated loops in or
before OpenMP version 4.5), variables are are implicitly linear. This
currently causes a problem with the existing linear clause
implementation. Hence, re-introduce TODO on the linear clause until the
linear clause lowering/translation are robust enough to handle such
cases as well.

Fixes https://github.com/llvm/llvm-project/issues/142935
---
 flang/lib/Lower/OpenMP/OpenMP.cpp         |  4 +--
 flang/test/Lower/OpenMP/wsloop-linear.f90 | 57 -------------------------------
 2 files changed, 2 insertions(+), 59 deletions(-)
 delete mode 100644 flang/test/Lower/OpenMP/wsloop-linear.f90

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ddf58fd87444..ebd1d038716e 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -1968,13 +1968,13 @@ static void genWsloopClauses(
     llvm::SmallVectorImpl<const semantics::Symbol *> &reductionSyms) {
   ClauseProcessor cp(converter, semaCtx, clauses);
   cp.processNowait(clauseOps);
-  cp.processLinear(clauseOps);
   cp.processOrder(clauseOps);
   cp.processOrdered(clauseOps);
   cp.processReduction(loc, clauseOps, reductionSyms);
   cp.processSchedule(stmtCtx, clauseOps);
 
-  cp.processTODO<clause::Allocate>(loc, llvm::omp::Directive::OMPD_do);
+  cp.processTODO<clause::Allocate, clause::Linear>(
+      loc, llvm::omp::Directive::OMPD_do);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/flang/test/Lower/OpenMP/wsloop-linear.f90 b/flang/test/Lower/OpenMP/wsloop-linear.f90
deleted file mode 100644
index b99677108be2..000000000000
--- a/flang/test/Lower/OpenMP/wsloop-linear.f90
+++ /dev/null
@@ -1,57 +0,0 @@
-! This test checks lowering of OpenMP DO Directive (Worksharing)
-! with linear clause
-
-! RUN: %flang_fc1 -fopenmp -emit-hlfir %s -o - 2>&1 | FileCheck %s
-
-!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFsimple_linearEx"}
-!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFsimple_linearEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[const:.*]] = arith.constant 1 : i32
-subroutine simple_linear
-    implicit none
-    integer :: x, y, i
-    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
-    !$omp do linear(x)
-    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
-    !CHECK: %[[const:.*]] = arith.constant 2 : i32
-    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32
-    do i = 1, 10
-        y = x + 2
-    end do
-    !$omp end do
-end subroutine
-
-
-!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_stepEx"}
-!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_stepEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-subroutine linear_step
-    implicit none
-    integer :: x, y, i
-    !CHECK: %[[const:.*]] = arith.constant 4 : i32
-    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[const]] : !fir.ref<i32>) {{.*}}
-    !$omp do linear(x:4)
-    !CHECK: %[[LOAD:.*]] = fir.load %[[X]]#0 : !fir.ref<i32>
-    !CHECK: %[[const:.*]] = arith.constant 2 : i32
-    !CHECK: %[[RESULT:.*]] = arith.addi %[[LOAD]], %[[const]] : i32   
-    do i = 1, 10
-        y = x + 2
-    end do
-    !$omp end do
-end subroutine
-
-!CHECK: %[[A_alloca:.*]] = fir.alloca i32 {bindc_name = "a", uniq_name = "_QFlinear_exprEa"}
-!CHECK: %[[A:.*]]:2 = hlfir.declare %[[A_alloca]] {uniq_name = "_QFlinear_exprEa"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-!CHECK: %[[X_alloca:.*]] = fir.alloca i32 {bindc_name = "x", uniq_name = "_QFlinear_exprEx"}
-!CHECK: %[[X:.*]]:2 = hlfir.declare %[[X_alloca]] {uniq_name = "_QFlinear_exprEx"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
-subroutine linear_expr
-    implicit none
-    integer :: x, y, i, a
-    !CHECK: %[[LOAD_A:.*]] = fir.load %[[A]]#0 : !fir.ref<i32>
-    !CHECK: %[[const:.*]] = arith.constant 4 : i32
-    !CHECK: %[[LINEAR_EXPR:.*]] = arith.addi %[[LOAD_A]], %[[const]] : i32
-    !CHECK: omp.wsloop linear(%[[X]]#0 = %[[LINEAR_EXPR]] : !fir.ref<i32>) {{.*}}
-    !$omp do linear(x:a+4)
-    do i = 1, 10
-        y = x + 2
-    end do
-    !$omp end do
-end subroutine
-- 
cgit v1.2.3


From a5b1093f782729014604f3208550de7400c518ac Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Fri, 20 Jun 2025 13:55:03 +0200
Subject: [MLIR] Add ReturnLike trait to memref.atomic_yield (#144932)

Without this, the yield isn't considered as the region terminator and
the dataflow framework does not consider it live.
---
 mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td |  3 +--
 mlir/test/Transforms/remove-dead-values.mlir     | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
index 481b14cdb462..b0fb5b078514 100644
--- a/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/IR/MemRefOps.td
@@ -1078,8 +1078,7 @@ def GenericAtomicRMWOp : MemRef_Op<"generic_atomic_rmw", [
 
 def AtomicYieldOp : MemRef_Op<"atomic_yield", [
       HasParent<"GenericAtomicRMWOp">,
-      Pure,
-      Terminator
+      Pure, Terminator, ReturnLike
     ]> {
   let summary = "yield operation for GenericAtomicRMWOp";
   let description = [{
diff --git a/mlir/test/Transforms/remove-dead-values.mlir b/mlir/test/Transforms/remove-dead-values.mlir
index 8c2a1cf7546f..e55a9160f5b3 100644
--- a/mlir/test/Transforms/remove-dead-values.mlir
+++ b/mlir/test/Transforms/remove-dead-values.mlir
@@ -510,3 +510,18 @@ module {
 // CHECK: %[[yield:.*]] = arith.addf %{{.*}}, %{{.*}} : f32
 // CHECK: linalg.yield %[[yield]] : f32
 // CHECK-NOT: arith.subf
+
+// -----
+
+// CHECK-LABEL: func.func @test_atomic_yield
+func.func @test_atomic_yield(%I: memref<10xf32>, %idx : index) {
+  // CHECK: memref.generic_atomic_rmw
+  %x = memref.generic_atomic_rmw %I[%idx] : memref<10xf32> {
+  ^bb0(%current_value : f32):
+    // CHECK: arith.constant
+    %c1 = arith.constant 1.0 : f32
+    // CHECK: memref.atomic_yield
+    memref.atomic_yield %c1 : f32
+  }
+  func.return
+}
-- 
cgit v1.2.3


From 5835f1e0a33afcae46a6ca4854373785eb3e7fd6 Mon Sep 17 00:00:00 2001
From: Tobias Stadler <mail@stadler-tobias.de>
Date: Fri, 20 Jun 2025 12:55:11 +0100
Subject: [AsmPrinter] Fix crash when remarks section is unsupported (#144724)

Emit a warning and bail out instead of segfault-ing when the current
object file format does not have support for emitting a remarks section.
---
 llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 12 +++++++++---
 llvm/test/CodeGen/X86/remarks-section.ll   |  5 +++++
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index a2c3b50b2467..403963f33b65 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2383,6 +2383,15 @@ void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) {
   if (!RS.needsSection())
     return;
 
+  MCSection *RemarksSection =
+      OutContext.getObjectFileInfo()->getRemarksSection();
+  if (!RemarksSection) {
+    OutContext.reportWarning(SMLoc(), "Current object file format does not "
+                                      "support remarks sections. Use the yaml "
+                                      "remark format instead.");
+    return;
+  }
+
   remarks::RemarkSerializer &RemarkSerializer = RS.getSerializer();
 
   std::optional<SmallString<128>> Filename;
@@ -2400,10 +2409,7 @@ void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) {
   MetaSerializer->emit();
 
   // Switch to the remarks section.
-  MCSection *RemarksSection =
-      OutContext.getObjectFileInfo()->getRemarksSection();
   OutStreamer->switchSection(RemarksSection);
-
   OutStreamer->emitBinaryData(Buf);
 }
 
diff --git a/llvm/test/CodeGen/X86/remarks-section.ll b/llvm/test/CodeGen/X86/remarks-section.ll
index e67c3579b759..2611e525aecf 100644
--- a/llvm/test/CodeGen/X86/remarks-section.ll
+++ b/llvm/test/CodeGen/X86/remarks-section.ll
@@ -5,6 +5,8 @@
 ; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=bitstream -remarks-section=false -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-BITSTREAM %s
 ; RUN: llc < %s -mtriple=x86_64-darwin --pass-remarks-format=yaml -remarks-section=true -pass-remarks-output=%/t.yaml | FileCheck --check-prefix=CHECK-DARWIN-OVERRIDE-YAML %s
 
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu --pass-remarks-format=bitstream -pass-remarks-output=%/t.yaml 2>&1 | FileCheck --check-prefix=CHECK-LINUX-DEFAULT-BITSTREAM %s
+
 ; CHECK-DARWIN: .section __LLVM,__remarks,regular,debug
 ; CHECK-DARWIN-NEXT: .byte
 
@@ -22,3 +24,6 @@
 define void @func1() {
   ret void
 }
+
+; Currently no ELF support for bitstream remarks
+; CHECK-LINUX-DEFAULT-BITSTREAM: warning: Current object file format does not support remarks sections. Use the yaml remark format instead.
-- 
cgit v1.2.3


From fd97dfbb78e3c9aea16873617b6d61b5b8a64474 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 20 Jun 2025 13:05:19 +0100
Subject: [LV] Don't mark ptrs as safe to speculate if fed by UB/poison op.
 (#143204)

Add additional checks before marking pointers safe to load
speculatively. If some computations feeding the pointer may trigger UB,
we cannot load the pointer speculatively, because we cannot compute the
address speculatively. The UB triggering instructions will be
predicated, but if the predicated block does not execute the result is
poison.

Similarly, we also cannot load the pointer speculatively if it may be
poison. The patch also checks if any of the operands defined outside the
loop may be poison when entering the loop. We *don't* need to check if
any operation inside the loop may produce poison due to flags, as those
will be dropped if needed.

There are some types of instructions inside the loop that can produce
poison independent of flags. Currently loads are also checked, not sure
if there's a convenient API to check for all such operands.

Fixes https://github.com/llvm/llvm-project/issues/142957.

PR: https://github.com/llvm/llvm-project/pull/143204
---
 .../Vectorize/LoopVectorizationLegality.cpp        |  41 +++++++
 .../X86/drop-poison-generating-flags.ll            | 124 ++++++++++++++-------
 ...renceable-info-from-assumption-constant-size.ll |  66 +++++------
 ...renceable-info-from-assumption-variable-size.ll |  16 +--
 ...oad-deref-pred-poison-ub-ops-feeding-pointer.ll |  66 ++++++-----
 5 files changed, 202 insertions(+), 111 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
index 0c4e5bb3d472..969d225c6ef2 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp
@@ -1491,10 +1491,51 @@ bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
     SmallVector<const SCEVPredicate *, 4> Predicates;
     for (Instruction &I : *BB) {
       LoadInst *LI = dyn_cast<LoadInst>(&I);
+
+      // Make sure we can execute all computations feeding into Ptr in the loop
+      // w/o triggering UB and that none of the out-of-loop operands are poison.
+      // We do not need to check if operations inside the loop can produce
+      // poison due to flags (e.g. due to an inbounds GEP going out of bounds),
+      // because flags will be dropped when executing them unconditionally.
+      // TODO: Results could be improved by considering poison-propagation
+      // properties of visited ops.
+      auto CanSpeculatePointerOp = [this](Value *Ptr) {
+        SmallVector<Value *> Worklist = {Ptr};
+        SmallPtrSet<Value *, 4> Visited;
+        while (!Worklist.empty()) {
+          Value *CurrV = Worklist.pop_back_val();
+          if (!Visited.insert(CurrV).second)
+            continue;
+
+          auto *CurrI = dyn_cast<Instruction>(CurrV);
+          if (!CurrI || !TheLoop->contains(CurrI)) {
+            // If operands from outside the loop may be poison then Ptr may also
+            // be poison.
+            if (!isGuaranteedNotToBePoison(CurrV, AC,
+                                           TheLoop->getLoopPredecessor()
+                                               ->getTerminator()
+                                               ->getIterator()))
+              return false;
+            continue;
+          }
+
+          // A loaded value may be poison, independent of any flags.
+          if (isa<LoadInst>(CurrI) && !isGuaranteedNotToBePoison(CurrV, AC))
+            return false;
+
+          // For other ops, assume poison can only be introduced via flags,
+          // which can be dropped.
+          if (!isa<PHINode>(CurrI) && !isSafeToSpeculativelyExecute(CurrI))
+            return false;
+          append_range(Worklist, CurrI->operands());
+        }
+        return true;
+      };
       // Pass the Predicates pointer to isDereferenceableAndAlignedInLoop so
       // that it will consider loops that need guarding by SCEV checks. The
       // vectoriser will generate these checks if we decide to vectorise.
       if (LI && !LI->getType()->isVectorTy() && !mustSuppressSpeculation(*LI) &&
+          CanSpeculatePointerOp(LI->getPointerOperand()) &&
           isDereferenceableAndAlignedInLoop(LI, TheLoop, SE, *DT, AC,
                                             &Predicates))
         SafePointers.insert(LI->getPointerOperand());
diff --git a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
index 53fd2ed43972..ff9cf682b6e9 100644
--- a/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
+++ b/llvm/test/Transforms/LoopVectorize/X86/drop-poison-generating-flags.ll
@@ -666,34 +666,54 @@ define void @pr70590_recipe_without_underlying_instr(i64 %n, ptr noalias %dst) {
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SREM_CONTINUE6:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_SREM_CONTINUE6]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
-; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_SREM_IF:.*]], label %[[PRED_SREM_CONTINUE:.*]]
-; CHECK:       [[PRED_SREM_IF]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE]]
-; CHECK:       [[PRED_SREM_CONTINUE]]:
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
+; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], poison
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i8> poison, i8 [[TMP6]], i32 0
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <4 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP24]], %[[PRED_LOAD_IF]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_SREM_IF1:.*]], label %[[PRED_SREM_CONTINUE2:.*]]
-; CHECK:       [[PRED_SREM_IF1]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE2]]
-; CHECK:       [[PRED_SREM_CONTINUE2]]:
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK:       [[PRED_LOAD_IF1]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP11:%.*]] = add i64 [[TMP10]], poison
+; CHECK-NEXT:    [[TMP25:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP26:%.*]] = load i8, ptr [[TMP25]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i8> [[TMP8]], i8 [[TMP26]], i32 1
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP29:%.*]] = phi <4 x i8> [ [[TMP8]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
-; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_SREM_IF3:.*]], label %[[PRED_SREM_CONTINUE4:.*]]
-; CHECK:       [[PRED_SREM_IF3]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE4]]
-; CHECK:       [[PRED_SREM_CONTINUE4]]:
+; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK:       [[PRED_LOAD_IF3]]:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], poison
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i8> [[TMP29]], i8 [[TMP20]], i32 2
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK:       [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi <4 x i8> [ [[TMP29]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP21]], %[[PRED_LOAD_IF3]] ]
 ; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_SREM_IF5:.*]], label %[[PRED_SREM_CONTINUE6]]
-; CHECK:       [[PRED_SREM_IF5]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE6]]
-; CHECK:       [[PRED_SREM_CONTINUE6]]:
-; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[INDEX]], poison
+; CHECK-NEXT:    br i1 [[TMP9]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_IF5]]:
+; CHECK-NEXT:    [[TMP30:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP12:%.*]] = add i64 [[TMP30]], poison
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr i8, ptr [[TMP13]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP14]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[TMP13]], align 1
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP27]], i32 3
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i8> zeroinitializer, <4 x i8> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i8, ptr [[DST]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr i8, ptr [[TMP15]], i32 0
@@ -743,34 +763,54 @@ define void @recipe_without_underlying_instr_lanes_used(i64 %n, ptr noalias %dst
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_SREM_CONTINUE6:.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_SREM_CONTINUE6]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6:.*]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_LOAD_CONTINUE6]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = icmp eq <4 x i64> [[VEC_IND]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    [[TMP1:%.*]] = xor <4 x i1> [[TMP0]], splat (i1 true)
 ; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i32 0
-; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_SREM_IF:.*]], label %[[PRED_SREM_CONTINUE:.*]]
-; CHECK:       [[PRED_SREM_IF]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE]]
-; CHECK:       [[PRED_SREM_CONTINUE]]:
+; CHECK-NEXT:    br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]]
+; CHECK:       [[PRED_LOAD_IF]]:
+; CHECK-NEXT:    [[TMP9:%.*]] = add i64 [[INDEX]], 0
+; CHECK-NEXT:    [[TMP16:%.*]] = add i64 [[TMP9]], poison
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP23]], align 1
+; CHECK-NEXT:    [[TMP24:%.*]] = insertelement <4 x i8> poison, i8 [[TMP6]], i32 0
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE]]
+; CHECK:       [[PRED_LOAD_CONTINUE]]:
+; CHECK-NEXT:    [[TMP25:%.*]] = phi <4 x i8> [ poison, %[[VECTOR_BODY]] ], [ [[TMP24]], %[[PRED_LOAD_IF]] ]
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <4 x i1> [[TMP1]], i32 1
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_SREM_IF1:.*]], label %[[PRED_SREM_CONTINUE2:.*]]
-; CHECK:       [[PRED_SREM_IF1]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE2]]
-; CHECK:       [[PRED_SREM_CONTINUE2]]:
+; CHECK-NEXT:    br i1 [[TMP3]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2:.*]]
+; CHECK:       [[PRED_LOAD_IF1]]:
+; CHECK-NEXT:    [[TMP26:%.*]] = add i64 [[INDEX]], 1
+; CHECK-NEXT:    [[TMP29:%.*]] = add i64 [[TMP26]], poison
+; CHECK-NEXT:    [[TMP30:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP29]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i8, ptr [[TMP30]], align 1
+; CHECK-NEXT:    [[TMP14:%.*]] = insertelement <4 x i8> [[TMP25]], i8 [[TMP13]], i32 1
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE2]]
+; CHECK:       [[PRED_LOAD_CONTINUE2]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = phi <4 x i8> [ [[TMP25]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP14]], %[[PRED_LOAD_IF1]] ]
 ; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <4 x i1> [[TMP1]], i32 2
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_SREM_IF3:.*]], label %[[PRED_SREM_CONTINUE4:.*]]
-; CHECK:       [[PRED_SREM_IF3]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE4]]
-; CHECK:       [[PRED_SREM_CONTINUE4]]:
+; CHECK-NEXT:    br i1 [[TMP4]], label %[[PRED_LOAD_IF3:.*]], label %[[PRED_LOAD_CONTINUE4:.*]]
+; CHECK:       [[PRED_LOAD_IF3]]:
+; CHECK-NEXT:    [[TMP17:%.*]] = add i64 [[INDEX]], 2
+; CHECK-NEXT:    [[TMP18:%.*]] = add i64 [[TMP17]], poison
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP18]]
+; CHECK-NEXT:    [[TMP20:%.*]] = load i8, ptr [[TMP19]], align 1
+; CHECK-NEXT:    [[TMP21:%.*]] = insertelement <4 x i8> [[TMP15]], i8 [[TMP20]], i32 2
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE4]]
+; CHECK:       [[PRED_LOAD_CONTINUE4]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = phi <4 x i8> [ [[TMP15]], %[[PRED_LOAD_CONTINUE2]] ], [ [[TMP21]], %[[PRED_LOAD_IF3]] ]
 ; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i1> [[TMP1]], i32 3
-; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_SREM_IF5:.*]], label %[[PRED_SREM_CONTINUE6]]
-; CHECK:       [[PRED_SREM_IF5]]:
-; CHECK-NEXT:    br label %[[PRED_SREM_CONTINUE6]]
-; CHECK:       [[PRED_SREM_CONTINUE6]]:
-; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[INDEX]], poison
+; CHECK-NEXT:    br i1 [[TMP5]], label %[[PRED_LOAD_IF5:.*]], label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_IF5]]:
+; CHECK-NEXT:    [[TMP31:%.*]] = add i64 [[INDEX]], 3
+; CHECK-NEXT:    [[TMP7:%.*]] = add i64 [[TMP31]], poison
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr [5 x i8], ptr @c, i64 0, i64 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr i8, ptr [[TMP8]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1
+; CHECK-NEXT:    [[TMP27:%.*]] = load i8, ptr [[TMP8]], align 1
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <4 x i8> [[TMP22]], i8 [[TMP27]], i32 3
+; CHECK-NEXT:    br label %[[PRED_LOAD_CONTINUE6]]
+; CHECK:       [[PRED_LOAD_CONTINUE6]]:
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = phi <4 x i8> [ [[TMP22]], %[[PRED_LOAD_CONTINUE4]] ], [ [[TMP28]], %[[PRED_LOAD_IF5]] ]
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP0]], <4 x i8> zeroinitializer, <4 x i8> [[WIDE_LOAD]]
 ; CHECK-NEXT:    [[PREDPHI7:%.*]] = select <4 x i1> [[TMP0]], <4 x i64> zeroinitializer, <4 x i64> poison
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <4 x i64> [[PREDPHI7]], i32 3
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
index dfae2d3f41d4..7a54519c7cdf 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll
@@ -3,9 +3,9 @@
 
 declare void @llvm.assume(i1)
 
-define void @deref_assumption_in_header_constant_trip_count(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_header_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -103,9 +103,9 @@ exit:
   ret void
 }
 
-define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -179,9 +179,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_too_small_in_header_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_header_constant_trip_count(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -279,9 +279,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_header_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_align_1(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -479,9 +479,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_header_constant_trip_count_align_not_known(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_align_not_known(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -579,9 +579,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_then_constant_trip_count(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_then_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_then_constant_trip_count(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -675,9 +675,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_latch_constant_trip_count(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_latch_constant_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_latch_constant_trip_count(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
@@ -777,9 +777,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_header_variable_trip_count(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) nofree nosync{
+define void @deref_assumption_in_header_variable_trip_count(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %N) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_header_variable_trip_count(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
 ; CHECK-NEXT:    br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -881,9 +881,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_preheader_constant_trip_count_align_1(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_preheader_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_1(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -958,9 +958,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_1(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 3999) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1053,9 +1053,9 @@ exit:
   ret void
 }
 
-define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @align_and_deref_assumption_in_preheader_constant_trip_count_align_4(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1131,9 +1131,9 @@ exit:
 }
 
 
-define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via_argument_attr(ptr noalias align 4 %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via_argument_attr(ptr noalias noundef align 4 %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via_argument_attr(
-; CHECK-SAME: ptr noalias align 4 [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef align 4 [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1145,9 +1145,9 @@ define void @deref_assumption_in_preheader_constant_trip_count_align_4_known_via
 ; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[TMP1]], i32 0
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP2]], align 4
 ; CHECK-NEXT:    [[TMP3:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP15:%.*]] = load <2 x i32>, ptr [[TMP5]], align 4
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <2 x i1> [[TMP3]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP15]]
 ; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[TMP0]]
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0
@@ -1208,9 +1208,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_constant_trip_count_align_4_not_known(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 4000) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1303,9 +1303,9 @@ exit:
   ret void
 }
 
-define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4(ptr noalias %a, ptr noalias %b, ptr noalias %c) nofree nosync{
+define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nofree nosync{
 ; CHECK-LABEL: define void @deref_assumption_too_small_in_preheader_constant_trip_count_align_4(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 3999) ]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
@@ -1399,9 +1399,9 @@ exit:
 }
 
 ; %a may be freed between the dereferenceable assumption and accesses.
-define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias %a, ptr noalias %b, ptr noalias %c) {
+define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) {
 ; CHECK-LABEL: define void @may_free_align_deref_assumption_in_header_constant_trip_count_loop_invariant_ptr(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4) ]
 ; CHECK-NEXT:    call void @may_free()
diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
index 6b46197295e3..ee65dc8cdcb1 100644
--- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-variable-size.ll
@@ -4,9 +4,9 @@
 declare void @llvm.assume(i1)
 
 ; %a is known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_non_constant_trip_count_access_i8(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
+define void @deref_assumption_in_preheader_non_constant_trip_count_access_i8(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_non_constant_trip_count_access_i8(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[N]]) ]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
@@ -86,9 +86,9 @@ exit:
 }
 
 ; %a is known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
+define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i64 [[N]], 4
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[MUL]]) ]
@@ -171,9 +171,9 @@ exit:
 
 
 ; %a is NOT known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_access_i32(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
+define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_too_small_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[N]]) ]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
@@ -253,9 +253,9 @@ exit:
 }
 
 ; %a is NOT known dereferenceable via assume for the whole loop.
-define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_access_i32(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
+define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_access_i32(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i64 %n) nofree nosync {
 ; CHECK-LABEL: define void @deref_assumption_in_preheader_too_small2_non_constant_trip_count_access_i32(
-; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
+; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i64 [[N:%.*]]) #[[ATTR1]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 100) ]
 ; CHECK-NEXT:    [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll b/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
index e27734755dfb..3373c6d5cb81 100644
--- a/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
+++ b/llvm/test/Transforms/LoopVectorize/load-deref-pred-poison-ub-ops-feeding-pointer.ll
@@ -7,7 +7,7 @@ target datalayout="p:16:16"
 
 ; Test cases for https://github.com/llvm/llvm-project/issues/142957
 
-; FIXME: Cannot speculatively execute %l, because %div may trigger UB and must be
+; Cannot speculatively execute %l, because %div may trigger UB and must be
 ; predicated.
 define void @ptr_depends_on_sdiv(ptr noalias %dst, i16 noundef %off) {
 ; CHECK-LABEL: define void @ptr_depends_on_sdiv(
@@ -26,36 +26,40 @@ define void @ptr_depends_on_sdiv(ptr noalias %dst, i16 noundef %off) {
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_SDIV_IF:.*]], label %[[PRED_SDIV_CONTINUE:.*]]
 ; CHECK:       [[PRED_SDIV_IF]]:
 ; CHECK-NEXT:    [[TMP2:%.*]] = sdiv i16 24316, [[OFF]]
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i32 0
 ; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE]]
 ; CHECK:       [[PRED_SDIV_CONTINUE]]:
-; CHECK-NEXT:    [[TMP3:%.*]] = phi i16 [ poison, %[[VECTOR_BODY]] ], [ [[TMP2]], %[[PRED_SDIV_IF]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = phi <2 x i16> [ poison, %[[VECTOR_BODY]] ], [ [[TMP3]], %[[PRED_SDIV_IF]] ]
 ; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_SDIV_IF1:.*]], label %[[PRED_SDIV_CONTINUE2:.*]]
 ; CHECK:       [[PRED_SDIV_IF1]]:
 ; CHECK-NEXT:    [[TMP18:%.*]] = sdiv i16 24316, [[OFF]]
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i16> [[TMP4]], i16 [[TMP18]], i32 1
 ; CHECK-NEXT:    br label %[[PRED_SDIV_CONTINUE2]]
 ; CHECK:       [[PRED_SDIV_CONTINUE2]]:
-; CHECK-NEXT:    [[TMP13:%.*]] = add i16 [[OFFSET_IDX]], 16383
-; CHECK-NEXT:    [[TMP7:%.*]] = shl i16 [[TMP3]], 14
-; CHECK-NEXT:    [[TMP8:%.*]] = sub i16 [[TMP13]], [[TMP7]]
-; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr i64, ptr @src, i16 [[TMP8]]
-; CHECK-NEXT:    [[TMP21:%.*]] = getelementptr i64, ptr [[TMP20]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP21]], align 1
+; CHECK-NEXT:    [[TMP8:%.*]] = phi <2 x i16> [ [[TMP4]], %[[PRED_SDIV_CONTINUE]] ], [ [[TMP7]], %[[PRED_SDIV_IF1]] ]
+; CHECK-NEXT:    [[TMP21:%.*]] = add <2 x i16> [[VEC_IND]], splat (i16 16383)
+; CHECK-NEXT:    [[TMP22:%.*]] = shl <2 x i16> [[TMP8]], splat (i16 14)
+; CHECK-NEXT:    [[TMP23:%.*]] = sub <2 x i16> [[TMP21]], [[TMP22]]
 ; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <2 x i1> [[TMP0]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP19]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i16> [[TMP23]], i32 0
+; CHECK-NEXT:    [[TMP24:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP13]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP24]], align 1
 ; CHECK-NEXT:    [[TMP10:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP10]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP11]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <2 x i1> [[TMP0]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP12]], label %[[PRED_STORE_IF3:.*]], label %[[PRED_STORE_CONTINUE4]]
 ; CHECK:       [[PRED_STORE_IF3]]:
+; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <2 x i16> [[TMP23]], i32 1
+; CHECK-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP25]]
+; CHECK-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP20]], align 1
 ; CHECK-NEXT:    [[TMP16:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP16]]
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
 ; CHECK-NEXT:    store i64 [[TMP15]], ptr [[TMP17]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE4]]
 ; CHECK:       [[PRED_STORE_CONTINUE4]]:
@@ -94,15 +98,17 @@ exit:
   ret void
 }
 
-; FIXME: Cannot speculatively execute %l, because %off may be poison.
+; Cannot speculatively execute %l, because %off may be poison.
 define void @ptr_depends_on_possibly_poison_value(ptr noalias %dst, i16 %off) {
 ; CHECK-LABEL: define void @ptr_depends_on_possibly_poison_value(
 ; CHECK-SAME: ptr noalias [[DST:%.*]], i16 [[OFF:%.*]]) {
 ; CHECK-NEXT:  [[ENTRY:.*:]]
 ; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i16 1, [[OFF]]
-; CHECK-NEXT:    [[TMP1:%.*]] = add i16 [[TMP0]], [[OFF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[OFF]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = sub <2 x i16> splat (i16 1), [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP1:%.*]] = add <2 x i16> [[TMP0]], [[BROADCAST_SPLAT]]
 ; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
 ; CHECK:       [[VECTOR_BODY]]:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE2:.*]] ]
@@ -110,25 +116,26 @@ define void @ptr_depends_on_possibly_poison_value(ptr noalias %dst, i16 %off) {
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
 ; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
-; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[OFFSET_IDX]], [[TMP1]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr i64, ptr @src, i16 [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP5]], align 1
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i16> [[VEC_IND]], [[TMP1]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i1> [[TMP2]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP6]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <2 x i16> [[TMP3]], i32 0
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP5]]
+; CHECK-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP14]], align 1
 ; CHECK-NEXT:    [[TMP7:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP7]]
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP9]], ptr [[TMP8]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i1> [[TMP2]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
 ; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <2 x i16> [[TMP3]], i32 1
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP15]]
+; CHECK-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP16]], align 1
 ; CHECK-NEXT:    [[TMP11:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP11]]
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
 ; CHECK-NEXT:    store i64 [[TMP13]], ptr [[TMP12]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
 ; CHECK:       [[PRED_STORE_CONTINUE2]]:
@@ -237,7 +244,7 @@ exit:
   ret void
 }
 
-; FIXME: Cannot speculatively execute %l, because %off may be poison.
+; Cannot speculatively execute %l, because %off may be poison.
 define void @ptr_depends_on_possibly_poison_value_from_load(ptr noalias %dst) {
 ; CHECK-LABEL: define void @ptr_depends_on_possibly_poison_value_from_load(
 ; CHECK-SAME: ptr noalias [[DST:%.*]]) {
@@ -251,28 +258,31 @@ define void @ptr_depends_on_possibly_poison_value_from_load(ptr noalias %dst) {
 ; CHECK-NEXT:    [[DOTCAST:%.*]] = trunc i32 [[INDEX]] to i16
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i16 9, [[DOTCAST]]
 ; CHECK-NEXT:    [[TMP0:%.*]] = load i16, ptr @src, align 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i16> [[BROADCAST_SPLATINSERT]], <2 x i16> poison, <2 x i32> zeroinitializer
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <2 x i16> [[VEC_IND]], splat (i16 10)
-; CHECK-NEXT:    [[TMP2:%.*]] = sub i16 1, [[TMP0]]
-; CHECK-NEXT:    [[TMP3:%.*]] = add i16 [[TMP2]], [[TMP0]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add i16 [[OFFSET_IDX]], [[TMP3]]
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr i64, ptr @src, i16 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP6]], align 1
+; CHECK-NEXT:    [[TMP2:%.*]] = sub <2 x i16> splat (i16 1), [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP3:%.*]] = add <2 x i16> [[TMP2]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP4:%.*]] = add <2 x i16> [[VEC_IND]], [[TMP3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0
 ; CHECK-NEXT:    br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
 ; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <2 x i16> [[TMP4]], i32 0
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP6]]
+; CHECK-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP15]], align 1
 ; CHECK-NEXT:    [[TMP8:%.*]] = add i16 [[OFFSET_IDX]], 0
 ; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 0
 ; CHECK-NEXT:    store i64 [[TMP10]], ptr [[TMP9]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
 ; CHECK:       [[PRED_STORE_CONTINUE]]:
 ; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1
 ; CHECK-NEXT:    br i1 [[TMP11]], label %[[PRED_STORE_IF1:.*]], label %[[PRED_STORE_CONTINUE2]]
 ; CHECK:       [[PRED_STORE_IF1]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <2 x i16> [[TMP4]], i32 1
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr @src, i16 [[TMP16]]
+; CHECK-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP17]], align 1
 ; CHECK-NEXT:    [[TMP12:%.*]] = add i16 [[OFFSET_IDX]], 1
 ; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[DST]], i16 [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <2 x i64> [[WIDE_LOAD]], i32 1
 ; CHECK-NEXT:    store i64 [[TMP14]], ptr [[TMP13]], align 1
 ; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE2]]
 ; CHECK:       [[PRED_STORE_CONTINUE2]]:
-- 
cgit v1.2.3


From 6c0ac888c59ca34a7f4e1dc2702b30e0db5cbac3 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Fri, 20 Jun 2025 14:32:56 +0200
Subject: [mlir][arith][NFC] Remove redundant trait declaration (#145007)

`Arith_Op` already declares the `ElementwiseMappable` traits, so they
don't have to be declared for `arith.select`.
---
 mlir/include/mlir/Dialect/Arith/IR/ArithOps.td | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
index 0518cac156eb..ef9ccb7e8794 100644
--- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
+++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td
@@ -1688,8 +1688,7 @@ def SelectOp : Arith_Op<"select", [Pure,
     AllTypesMatch<["true_value", "false_value", "result"]>,
     BooleanConditionOrMatchingShape<"condition", "result">,
     DeclareOpInterfaceMethods<InferIntRangeInterface, ["inferResultRangesFromOptional"]>,
-    DeclareOpInterfaceMethods<SelectLikeOpInterface>,
-  ] # ElementwiseMappable.traits> {
+    DeclareOpInterfaceMethods<SelectLikeOpInterface>]> {
   let summary = "select operation";
   let description = [{
     The `arith.select` operation chooses one value based on a binary condition
-- 
cgit v1.2.3


From 6edf2eb36470b623597668a7a97153544f568a40 Mon Sep 17 00:00:00 2001
From: Artemiy Bulavin <artemiyb@graphcore.ai>
Date: Fri, 20 Jun 2025 13:45:17 +0100
Subject: [MLIR] Print more user-friendly error message when generating local
 reproducer and threading is enabled (#144905)

---
 mlir/lib/Pass/PassManagerOptions.cpp                    | 8 ++++++++
 mlir/test/mlir-opt/local-reproducer-with-threading.mlir | 7 +++++++
 2 files changed, 15 insertions(+)
 create mode 100644 mlir/test/mlir-opt/local-reproducer-with-threading.mlir

diff --git a/mlir/lib/Pass/PassManagerOptions.cpp b/mlir/lib/Pass/PassManagerOptions.cpp
index dd119a75f406..305bf72bb479 100644
--- a/mlir/lib/Pass/PassManagerOptions.cpp
+++ b/mlir/lib/Pass/PassManagerOptions.cpp
@@ -146,6 +146,14 @@ LogicalResult mlir::applyPassManagerCLOptions(PassManager &pm) {
   if (!options.isConstructed())
     return failure();
 
+  if (options->reproducerFile.getNumOccurrences() && options->localReproducer &&
+      pm.getContext()->isMultithreadingEnabled()) {
+    emitError(UnknownLoc::get(pm.getContext()))
+        << "Local crash reproduction may not be used without disabling "
+           "mutli-threading first.";
+    return failure();
+  }
+
   // Generate a reproducer on crash/failure.
   if (options->reproducerFile.getNumOccurrences())
     pm.enableCrashReproducerGeneration(options->reproducerFile,
diff --git a/mlir/test/mlir-opt/local-reproducer-with-threading.mlir b/mlir/test/mlir-opt/local-reproducer-with-threading.mlir
new file mode 100644
index 000000000000..8e94f4edb91b
--- /dev/null
+++ b/mlir/test/mlir-opt/local-reproducer-with-threading.mlir
@@ -0,0 +1,7 @@
+// Test that attempting to create a local crash reproducer without disabling threading
+// prints an error from the pass manager (as opposed to crashing with a stack trace).
+
+// RUN: mlir-opt --verify-diagnostics --mlir-pass-pipeline-local-reproducer \
+// RUN:          --mlir-pass-pipeline-crash-reproducer=%t %s
+
+// expected-error@unknown {{Local crash reproduction may not be used without disabling mutli-threading first.}}
-- 
cgit v1.2.3


From b334ffd4f4e70a404a8572b132184583a9d8623a Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Fri, 20 Jun 2025 14:01:15 +0100
Subject: [VPlan] Refine return types in VPBuilder (NFC) (#108858)

---
 .../Vectorize/LoopVectorizationPlanner.h           | 47 +++++++++++-----------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
index d17c64a778e8..144f35e10132 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
@@ -190,37 +190,38 @@ public:
         new VPInstruction(Opcode, Operands, WrapFlags, DL, Name));
   }
 
-  VPValue *createNot(VPValue *Operand, DebugLoc DL = DebugLoc::getUnknown(),
-                     const Twine &Name = "") {
+  VPInstruction *createNot(VPValue *Operand,
+                           DebugLoc DL = DebugLoc::getUnknown(),
+                           const Twine &Name = "") {
     return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
   }
 
-  VPValue *createAnd(VPValue *LHS, VPValue *RHS,
-                     DebugLoc DL = DebugLoc::getUnknown(),
-                     const Twine &Name = "") {
+  VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
+                           DebugLoc DL = DebugLoc::getUnknown(),
+                           const Twine &Name = "") {
     return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
   }
 
-  VPValue *createOr(VPValue *LHS, VPValue *RHS,
-                    DebugLoc DL = DebugLoc::getUnknown(),
-                    const Twine &Name = "") {
+  VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
+                          DebugLoc DL = DebugLoc::getUnknown(),
+                          const Twine &Name = "") {
 
     return tryInsertInstruction(new VPInstruction(
         Instruction::BinaryOps::Or, {LHS, RHS},
         VPRecipeWithIRFlags::DisjointFlagsTy(false), DL, Name));
   }
 
-  VPValue *createLogicalAnd(VPValue *LHS, VPValue *RHS,
-                            DebugLoc DL = DebugLoc::getUnknown(),
-                            const Twine &Name = "") {
+  VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
+                                  DebugLoc DL = DebugLoc::getUnknown(),
+                                  const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
   }
 
-  VPValue *createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
-                        DebugLoc DL = DebugLoc::getUnknown(),
-                        const Twine &Name = "",
-                        std::optional<FastMathFlags> FMFs = std::nullopt) {
+  VPInstruction *
+  createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
+               DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
+               std::optional<FastMathFlags> FMFs = std::nullopt) {
     auto *Select =
         FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
                                  *FMFs, DL, Name)
@@ -232,9 +233,9 @@ public:
   /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
   /// and \p B.
   /// TODO: add createFCmp when needed.
-  VPValue *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
-                      DebugLoc DL = DebugLoc::getUnknown(),
-                      const Twine &Name = "") {
+  VPInstruction *createICmp(CmpInst::Predicate Pred, VPValue *A, VPValue *B,
+                            DebugLoc DL = DebugLoc::getUnknown(),
+                            const Twine &Name = "") {
     assert(Pred >= CmpInst::FIRST_ICMP_PREDICATE &&
            Pred <= CmpInst::LAST_ICMP_PREDICATE && "invalid predicate");
     return tryInsertInstruction(
@@ -248,16 +249,16 @@ public:
         new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
                           GEPNoWrapFlags::none(), DL, Name));
   }
-  VPValue *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset,
-                                DebugLoc DL = DebugLoc::getUnknown(),
-                                const Twine &Name = "") {
+  VPInstruction *createInBoundsPtrAdd(VPValue *Ptr, VPValue *Offset,
+                                      DebugLoc DL = DebugLoc::getUnknown(),
+                                      const Twine &Name = "") {
     return tryInsertInstruction(
         new VPInstruction(VPInstruction::PtrAdd, {Ptr, Offset},
                           GEPNoWrapFlags::inBounds(), DL, Name));
   }
 
-  VPInstruction *createScalarPhi(ArrayRef<VPValue *> IncomingValues,
-                                 DebugLoc DL, const Twine &Name = "") {
+  VPPhi *createScalarPhi(ArrayRef<VPValue *> IncomingValues, DebugLoc DL,
+                         const Twine &Name = "") {
     return tryInsertInstruction(new VPPhi(IncomingValues, DL, Name));
   }
 
-- 
cgit v1.2.3


From c8c4bd1ebc6e4451dc835a77bacdbe6a0467f219 Mon Sep 17 00:00:00 2001
From: Ramkumar Ramachandra <ramkumar.ramachandra@codasip.com>
Date: Fri, 20 Jun 2025 14:01:48 +0100
Subject: [LV] Stengthen loop-invariance checks in isPredicatedInst (#140744)

Check loop-invariance against SCEV as well.
---
 llvm/lib/Analysis/LoopAccessAnalysis.cpp           |   4 +-
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    |   4 +-
 .../test/Transforms/LoopVectorize/RISCV/pr88802.ll | 114 +--------
 .../LoopVectorize/predicatedinst-loop-invariant.ll | 263 +++++++++++++++++++++
 4 files changed, 269 insertions(+), 116 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll

diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 00cdb66d8b77..94b9fe958126 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -2764,8 +2764,8 @@ LoopAccessInfo::recordAnalysis(StringRef RemarkName, const Instruction *I) {
 
 bool LoopAccessInfo::isInvariant(Value *V) const {
   auto *SE = PSE->getSE();
-  // TODO: Is this really what we want? Even without FP SCEV, we may want some
-  // trivially loop-invariant FP values to be considered invariant.
+  if (TheLoop->isLoopInvariant(V))
+    return true;
   if (!SE->isSCEVable(V->getType()))
     return false;
   const SCEV *S = SE->getSCEV(V);
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index d9f53c4146c2..88b2ffba1b79 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -3104,14 +3104,14 @@ bool LoopVectorizationCostModel::isPredicatedInst(Instruction *I) const {
     // is correct.  The easiest form of the later is to require that all values
     // stored are the same.
     return !(Legal->isInvariant(getLoadStorePointerOperand(I)) &&
-             TheLoop->isLoopInvariant(cast<StoreInst>(I)->getValueOperand()));
+             Legal->isInvariant(cast<StoreInst>(I)->getValueOperand()));
   }
   case Instruction::UDiv:
   case Instruction::SDiv:
   case Instruction::SRem:
   case Instruction::URem:
     // If the divisor is loop-invariant no predication is needed.
-    return !TheLoop->isLoopInvariant(I->getOperand(1));
+    return !Legal->isInvariant(I->getOperand(1));
   }
 }
 
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
index 51a8b451dffd..a1201dcfbdf5 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/pr88802.ll
@@ -17,126 +17,16 @@ define void @test(ptr %p, i64 %a, i8 %b) {
 ; CHECK-NEXT:    [[TMP3:%.*]] = zext <16 x i8> [[BROADCAST_SPLAT]] to <16 x i32>
 ; CHECK-NEXT:    br label [[FOR_COND:%.*]]
 ; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE8:%.*]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE8]] ]
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[FOR_COND]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <16 x i32> [ <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[FOR_COND]] ]
 ; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i32(i32 [[INDEX]], i32 9)
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp sge <16 x i32> [[VEC_IND]], splat (i32 2)
 ; CHECK-NEXT:    [[TMP5:%.*]] = select <16 x i1> [[ACTIVE_LANE_MASK]], <16 x i1> [[TMP4]], <16 x i1> zeroinitializer
 ; CHECK-NEXT:    [[PREDPHI:%.*]] = select <16 x i1> [[TMP5]], <16 x i32> [[TMP2]], <16 x i32> [[TMP3]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = shl <16 x i32> [[PREDPHI]], splat (i32 8)
 ; CHECK-NEXT:    [[TMP8:%.*]] = trunc <16 x i32> [[TMP6]] to <16 x i8>
-; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 0
-; CHECK-NEXT:    br i1 [[TMP9]], label [[PRED_STORE_IF:%.*]], label [[VECTOR_BODY:%.*]]
-; CHECK:       pred.store.if:
-; CHECK-NEXT:    [[TMP19:%.*]] = extractelement <16 x i8> [[TMP8]], i32 0
-; CHECK-NEXT:    store i8 [[TMP19]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[VECTOR_BODY]]
-; CHECK:       pred.store.continue:
-; CHECK-NEXT:    [[CMP_N:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 1
-; CHECK-NEXT:    br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH:%.*]]
-; CHECK:       pred.store.if3:
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <16 x i8> [[TMP8]], i32 1
-; CHECK-NEXT:    store i8 [[TMP12]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[SCALAR_PH]]
-; CHECK:       pred.store.continue4:
-; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 2
-; CHECK-NEXT:    br i1 [[TMP13]], label [[PRED_STORE_IF5:%.*]], label [[PRED_STORE_CONTINUE6:%.*]]
-; CHECK:       pred.store.if5:
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <16 x i8> [[TMP8]], i32 2
-; CHECK-NEXT:    store i8 [[TMP14]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE6]]
-; CHECK:       pred.store.continue6:
-; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 3
-; CHECK-NEXT:    br i1 [[TMP15]], label [[PRED_STORE_IF7:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
-; CHECK:       pred.store.if7:
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <16 x i8> [[TMP8]], i32 3
-; CHECK-NEXT:    store i8 [[TMP16]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE9]]
-; CHECK:       pred.store.continue8:
-; CHECK-NEXT:    [[TMP17:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 4
-; CHECK-NEXT:    br i1 [[TMP17]], label [[PRED_STORE_IF9:%.*]], label [[PRED_STORE_CONTINUE10:%.*]]
-; CHECK:       pred.store.if9:
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <16 x i8> [[TMP8]], i32 4
-; CHECK-NEXT:    store i8 [[TMP18]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE10]]
-; CHECK:       pred.store.continue10:
-; CHECK-NEXT:    [[TMP41:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 5
-; CHECK-NEXT:    br i1 [[TMP41]], label [[PRED_STORE_IF11:%.*]], label [[PRED_STORE_CONTINUE12:%.*]]
-; CHECK:       pred.store.if11:
-; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <16 x i8> [[TMP8]], i32 5
-; CHECK-NEXT:    store i8 [[TMP20]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE12]]
-; CHECK:       pred.store.continue12:
-; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 6
-; CHECK-NEXT:    br i1 [[TMP21]], label [[PRED_STORE_IF13:%.*]], label [[PRED_STORE_CONTINUE14:%.*]]
-; CHECK:       pred.store.if13:
-; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <16 x i8> [[TMP8]], i32 6
-; CHECK-NEXT:    store i8 [[TMP22]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE14]]
-; CHECK:       pred.store.continue14:
-; CHECK-NEXT:    [[TMP23:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 7
-; CHECK-NEXT:    br i1 [[TMP23]], label [[PRED_STORE_IF15:%.*]], label [[PRED_STORE_CONTINUE16:%.*]]
-; CHECK:       pred.store.if15:
-; CHECK-NEXT:    [[TMP24:%.*]] = extractelement <16 x i8> [[TMP8]], i32 7
-; CHECK-NEXT:    store i8 [[TMP24]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE16]]
-; CHECK:       pred.store.continue16:
-; CHECK-NEXT:    [[TMP25:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 8
-; CHECK-NEXT:    br i1 [[TMP25]], label [[PRED_STORE_IF17:%.*]], label [[PRED_STORE_CONTINUE18:%.*]]
-; CHECK:       pred.store.if17:
-; CHECK-NEXT:    [[TMP26:%.*]] = extractelement <16 x i8> [[TMP8]], i32 8
-; CHECK-NEXT:    store i8 [[TMP26]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE18]]
-; CHECK:       pred.store.continue18:
-; CHECK-NEXT:    [[TMP27:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 9
-; CHECK-NEXT:    br i1 [[TMP27]], label [[PRED_STORE_IF19:%.*]], label [[PRED_STORE_CONTINUE20:%.*]]
-; CHECK:       pred.store.if19:
-; CHECK-NEXT:    [[TMP28:%.*]] = extractelement <16 x i8> [[TMP8]], i32 9
-; CHECK-NEXT:    store i8 [[TMP28]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE20]]
-; CHECK:       pred.store.continue20:
-; CHECK-NEXT:    [[TMP29:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 10
-; CHECK-NEXT:    br i1 [[TMP29]], label [[PRED_STORE_IF21:%.*]], label [[PRED_STORE_CONTINUE22:%.*]]
-; CHECK:       pred.store.if21:
-; CHECK-NEXT:    [[TMP30:%.*]] = extractelement <16 x i8> [[TMP8]], i32 10
-; CHECK-NEXT:    store i8 [[TMP30]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE22]]
-; CHECK:       pred.store.continue22:
-; CHECK-NEXT:    [[TMP31:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 11
-; CHECK-NEXT:    br i1 [[TMP31]], label [[PRED_STORE_IF23:%.*]], label [[PRED_STORE_CONTINUE24:%.*]]
-; CHECK:       pred.store.if23:
-; CHECK-NEXT:    [[TMP32:%.*]] = extractelement <16 x i8> [[TMP8]], i32 11
-; CHECK-NEXT:    store i8 [[TMP32]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE24]]
-; CHECK:       pred.store.continue24:
-; CHECK-NEXT:    [[TMP33:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 12
-; CHECK-NEXT:    br i1 [[TMP33]], label [[PRED_STORE_IF25:%.*]], label [[PRED_STORE_CONTINUE26:%.*]]
-; CHECK:       pred.store.if25:
-; CHECK-NEXT:    [[TMP34:%.*]] = extractelement <16 x i8> [[TMP8]], i32 12
-; CHECK-NEXT:    store i8 [[TMP34]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE26]]
-; CHECK:       pred.store.continue26:
-; CHECK-NEXT:    [[TMP35:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 13
-; CHECK-NEXT:    br i1 [[TMP35]], label [[PRED_STORE_IF27:%.*]], label [[PRED_STORE_CONTINUE28:%.*]]
-; CHECK:       pred.store.if27:
-; CHECK-NEXT:    [[TMP36:%.*]] = extractelement <16 x i8> [[TMP8]], i32 13
-; CHECK-NEXT:    store i8 [[TMP36]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE28]]
-; CHECK:       pred.store.continue28:
-; CHECK-NEXT:    [[TMP37:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 14
-; CHECK-NEXT:    br i1 [[TMP37]], label [[PRED_STORE_IF29:%.*]], label [[PRED_STORE_CONTINUE30:%.*]]
-; CHECK:       pred.store.if29:
-; CHECK-NEXT:    [[TMP38:%.*]] = extractelement <16 x i8> [[TMP8]], i32 14
-; CHECK-NEXT:    store i8 [[TMP38]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE30]]
-; CHECK:       pred.store.continue30:
-; CHECK-NEXT:    [[TMP39:%.*]] = extractelement <16 x i1> [[ACTIVE_LANE_MASK]], i32 15
-; CHECK-NEXT:    br i1 [[TMP39]], label [[PRED_STORE_IF31:%.*]], label [[PRED_STORE_CONTINUE8]]
-; CHECK:       pred.store.if31:
 ; CHECK-NEXT:    [[TMP40:%.*]] = extractelement <16 x i8> [[TMP8]], i32 15
 ; CHECK-NEXT:    store i8 [[TMP40]], ptr [[P]], align 1
-; CHECK-NEXT:    br label [[PRED_STORE_CONTINUE8]]
-; CHECK:       pred.store.continue32:
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 16
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <16 x i32> [[VEC_IND]], splat (i32 16)
 ; CHECK-NEXT:    br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]]
diff --git a/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
new file mode 100644
index 000000000000..0a975108edee
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/predicatedinst-loop-invariant.ll
@@ -0,0 +1,263 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5
+; RUN: opt -passes=loop-vectorize -force-vector-width=4 -S %s | FileCheck %s
+
+define void @loop_invariant_store(ptr %p, i64 %a, i8 %b) {
+; CHECK-LABEL: define void @loop_invariant_store(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[A]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT2]], splat (i64 48)
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[TMP0]], splat (i64 52)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sge <4 x i32> [[VEC_IND]], splat (i32 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <4 x i32> [[PREDPHI]], splat (i32 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i8>
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <4 x i8> [[TMP8]], i32 3
+; CHECK-NEXT:    store i8 [[TMP9]], ptr [[P]], align 1
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4)
+; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[ADD:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[ADD]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i32 [[IV]], 2
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[A]], 48
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i64 [[SHL]], 52
+; CHECK-NEXT:    [[TRUNC_I32:%.*]] = trunc i64 [[ASHR]] to i32
+; CHECK-NEXT:    br i1 [[CMP_SLT]], label %[[COND_FALSE:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[B]] to i32
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[TRUNC_I32]], %[[LOOP_HEADER]] ], [ [[ZEXT]], %[[COND_FALSE]] ]
+; CHECK-NEXT:    [[SHL_I32:%.*]] = shl i32 [[COND]], 8
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHL_I32]] to i8
+; CHECK-NEXT:    store i8 [[TRUNC]], ptr [[P]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[IV]], 8
+; CHECK-NEXT:    br i1 [[CMP]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                      ; preds = %loop.latch, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %iv.next = add i32 %iv, 1
+  %cmp.slt = icmp slt i32 %iv, 2
+  %shl = shl i64 %a, 48
+  %ashr = ashr i64 %shl, 52
+  %trunc.i32 = trunc i64 %ashr to i32
+  br i1 %cmp.slt, label %cond.false, label %loop.latch
+
+cond.false:                                       ; preds = %loop.header
+  %zext = zext i8 %b to i32
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %cond.false, %loop.header
+  %cond = phi i32 [ %trunc.i32, %loop.header ], [ %zext, %cond.false ]
+  %shl.i32 = shl i32 %cond, 8
+  %trunc = trunc i32 %shl.i32 to i8
+  store i8 %trunc, ptr %p, align 1
+  %exitcond = icmp slt i32 %iv, 8
+  br i1 %exitcond, label %loop.header, label %exit
+
+exit:                                             ; preds = %loop.latch
+  ret void
+}
+
+define void @loop_invariant_srem(ptr %p, i64 %a, i8 %b) {
+; CHECK-LABEL: define void @loop_invariant_srem(
+; CHECK-SAME: ptr [[P:%.*]], i64 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i8> poison, i8 [[B]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i8> [[BROADCAST_SPLATINSERT]], <4 x i8> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[A]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP0:%.*]] = shl <4 x i64> [[BROADCAST_SPLAT2]], splat (i64 48)
+; CHECK-NEXT:    [[TMP1:%.*]] = ashr <4 x i64> [[TMP0]], splat (i64 52)
+; CHECK-NEXT:    [[TMP2:%.*]] = trunc <4 x i64> [[TMP1]] to <4 x i32>
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <4 x i8> [[BROADCAST_SPLAT]] to <4 x i32>
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_STORE_CONTINUE10:.*]] ]
+; CHECK-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i8> [ <i8 0, i8 1, i8 2, i8 3>, %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[PRED_STORE_CONTINUE10]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[INDEX]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IND:%.*]] = add <4 x i32> [[BROADCAST_SPLAT4]], <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], splat (i32 8)
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp sge <4 x i8> [[VEC_IND1]], splat (i8 2)
+; CHECK-NEXT:    [[TMP6:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer
+; CHECK-NEXT:    [[PREDPHI:%.*]] = select <4 x i1> [[TMP6]], <4 x i32> [[TMP2]], <4 x i32> [[TMP3]]
+; CHECK-NEXT:    [[TMP7:%.*]] = shl <4 x i32> [[PREDPHI]], splat (i32 8)
+; CHECK-NEXT:    [[TMP8:%.*]] = trunc <4 x i32> [[TMP7]] to <4 x i8>
+; CHECK-NEXT:    [[TMP11:%.*]] = srem <4 x i8> [[VEC_IND1]], [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
+; CHECK-NEXT:    br i1 [[TMP10]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
+; CHECK:       [[PRED_STORE_IF]]:
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <4 x i8> [[TMP11]], i32 0
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr i32, ptr [[P]], i8 [[TMP13]]
+; CHECK-NEXT:    store i32 4, ptr [[TMP12]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE]]
+; CHECK:       [[PRED_STORE_CONTINUE]]:
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
+; CHECK-NEXT:    br i1 [[TMP14]], label %[[PRED_STORE_IF5:.*]], label %[[PRED_STORE_CONTINUE6:.*]]
+; CHECK:       [[PRED_STORE_IF5]]:
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <4 x i8> [[TMP11]], i32 1
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr i32, ptr [[P]], i8 [[TMP16]]
+; CHECK-NEXT:    store i32 4, ptr [[TMP15]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE6]]
+; CHECK:       [[PRED_STORE_CONTINUE6]]:
+; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
+; CHECK-NEXT:    br i1 [[TMP18]], label %[[PRED_STORE_IF7:.*]], label %[[PRED_STORE_CONTINUE8:.*]]
+; CHECK:       [[PRED_STORE_IF7]]:
+; CHECK-NEXT:    [[TMP20:%.*]] = extractelement <4 x i8> [[TMP11]], i32 2
+; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr i32, ptr [[P]], i8 [[TMP20]]
+; CHECK-NEXT:    store i32 4, ptr [[TMP19]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE8]]
+; CHECK:       [[PRED_STORE_CONTINUE8]]:
+; CHECK-NEXT:    [[TMP22:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
+; CHECK-NEXT:    br i1 [[TMP22]], label %[[PRED_STORE_IF9:.*]], label %[[PRED_STORE_CONTINUE10]]
+; CHECK:       [[PRED_STORE_IF9]]:
+; CHECK-NEXT:    [[TMP21:%.*]] = extractelement <4 x i8> [[TMP11]], i32 3
+; CHECK-NEXT:    [[TMP23:%.*]] = getelementptr i32, ptr [[P]], i8 [[TMP21]]
+; CHECK-NEXT:    store i32 4, ptr [[TMP23]], align 4
+; CHECK-NEXT:    br label %[[PRED_STORE_CONTINUE10]]
+; CHECK:       [[PRED_STORE_CONTINUE10]]:
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <4 x i8> [[VEC_IND1]], splat (i8 4)
+; CHECK-NEXT:    [[TMP26:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i8 [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i8 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i8 [[IV]], 1
+; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i8 [[IV]], 2
+; CHECK-NEXT:    [[SHL:%.*]] = shl i64 [[A]], 48
+; CHECK-NEXT:    [[ASHR:%.*]] = ashr i64 [[SHL]], 52
+; CHECK-NEXT:    [[TRUNC_I32:%.*]] = trunc i64 [[ASHR]] to i32
+; CHECK-NEXT:    br i1 [[CMP_SLT]], label %[[COND_FALSE:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    [[ZEXT:%.*]] = zext i8 [[B]] to i32
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[TRUNC_I32]], %[[LOOP_HEADER]] ], [ [[ZEXT]], %[[COND_FALSE]] ]
+; CHECK-NEXT:    [[SHL_I32:%.*]] = shl i32 [[COND]], 8
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i32 [[SHL_I32]] to i8
+; CHECK-NEXT:    [[REM:%.*]] = srem i8 [[IV]], [[TRUNC]]
+; CHECK-NEXT:    [[GEP_P_REM:%.*]] = getelementptr i32, ptr [[P]], i8 [[REM]]
+; CHECK-NEXT:    store i32 4, ptr [[GEP_P_REM]], align 4
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i8 [[IV]], 8
+; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %loop.header
+
+loop.header:                                      ; preds = %loop.latch, %entry
+  %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %iv.next = add i8 %iv, 1
+  %cmp.slt = icmp slt i8 %iv, 2
+  %shl = shl i64 %a, 48
+  %ashr = ashr i64 %shl, 52
+  %trunc.i32 = trunc i64 %ashr to i32
+  br i1 %cmp.slt, label %cond.false, label %loop.latch
+
+cond.false:                                       ; preds = %loop.header
+  %zext = zext i8 %b to i32
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %cond.false, %loop.header
+  %cond = phi i32 [ %trunc.i32, %loop.header ], [ %zext, %cond.false ]
+  %shl.i32 = shl i32 %cond, 8
+  %trunc = trunc i32 %shl.i32 to i8
+  %rem = srem i8 %iv, %trunc
+  %gep.p.rem = getelementptr i32, ptr %p, i8 %rem
+  store i32 4, ptr %gep.p.rem
+  %ec = icmp eq i8 %iv, 8
+  br i1 %ec, label %exit, label %loop.header
+
+exit:                                             ; preds = %loop.latch
+  ret void
+}
+
+define void @loop_invariant_float_store(ptr %p, i32 %a) {
+; CHECK-LABEL: define void @loop_invariant_float_store(
+; CHECK-SAME: ptr [[P:%.*]], i32 [[A:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[TMP10:%.*]] = sitofp i32 [[A]] to float
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK:       [[VECTOR_PH]]:
+; CHECK-NEXT:    br label %[[VECTOR_BODY:.*]]
+; CHECK:       [[VECTOR_BODY]]:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; CHECK-NEXT:    store float [[TMP10]], ptr [[P]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
+; CHECK-NEXT:    br i1 [[TMP17]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK:       [[MIDDLE_BLOCK]]:
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[SCALAR_PH]]:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 0, %[[ENTRY]] ]
+; CHECK-NEXT:    br label %[[LOOP_HEADER:.*]]
+; CHECK:       [[LOOP_HEADER]]:
+; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ]
+; CHECK-NEXT:    [[IV_NEXT]] = add i32 [[IV]], 1
+; CHECK-NEXT:    [[CMP_SLT:%.*]] = icmp slt i32 [[IV]], 2
+; CHECK-NEXT:    br i1 [[CMP_SLT]], label %[[COND_FALSE:.*]], label %[[LOOP_LATCH]]
+; CHECK:       [[COND_FALSE]]:
+; CHECK-NEXT:    br label %[[LOOP_LATCH]]
+; CHECK:       [[LOOP_LATCH]]:
+; CHECK-NEXT:    store float [[TMP10]], ptr [[P]], align 4
+; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp slt i32 [[IV]], 8
+; CHECK-NEXT:    br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %a.conv = sitofp i32 %a to float
+  br label %loop.header
+
+loop.header:                                      ; preds = %loop.latch, %entry
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ]
+  %iv.next = add i32 %iv, 1
+  %cmp.slt = icmp slt i32 %iv, 2
+  br i1 %cmp.slt, label %cond.false, label %loop.latch
+
+cond.false:                                       ; preds = %loop.header
+  br label %loop.latch
+
+loop.latch:                                       ; preds = %cond.false, %loop.header
+  store float %a.conv, ptr %p
+  %exitcond = icmp slt i32 %iv, 8
+  br i1 %exitcond, label %loop.header, label %exit
+
+exit:                                             ; preds = %loop.latch
+  ret void
+}
-- 
cgit v1.2.3


From 349f8d67d4ee2e7a6045b02f6aea0a72165404b1 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 20 Jun 2025 08:09:36 -0500
Subject: [flang][OpenMP] Skip runtime mapping with no offload targets
 (#144534)

When no offload targets are specified flang will ignore "target"
constructs, but not "target data" constructs. This patch makes the
behavior consistent across all offload-related operations.

While ignoring "target" may produce semantically incorrect code, it may
still be a useful debugging tool.
---
 flang/test/Lower/ignore-target-data.f90            |  30 ++
 .../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp   |   6 +
 mlir/test/Target/LLVMIR/omptarget-llvm.mlir        | 348 +++++++++++----------
 .../Target/LLVMIR/omptargetdata-nowait-llvm.mlir   |  42 +--
 .../Target/LLVMIR/openmp-data-target-device.mlir   |   2 +-
 5 files changed, 245 insertions(+), 183 deletions(-)
 create mode 100644 flang/test/Lower/ignore-target-data.f90

diff --git a/flang/test/Lower/ignore-target-data.f90 b/flang/test/Lower/ignore-target-data.f90
new file mode 100644
index 000000000000..08947c137090
--- /dev/null
+++ b/flang/test/Lower/ignore-target-data.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=NORT
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=LLVM
+
+!Make sure that there are no calls to the mapper.
+!NORT-NOT: call{{.*}}__tgt_target_data_begin_mapper
+!NORT-NOT: call{{.*}}__tgt_target_data_end_mapper
+
+!Make sure we generate the body
+!LLVM: define internal void @_QFPf(ptr %[[A0:[0-9]+]], ptr %[[A1:[0-9]+]]) {
+!LLVM:   %[[V0:[0-9]+]] = load i32, ptr %[[A0]], align 4
+!LLVM:   %[[V1:[0-9]+]] = load i32, ptr %[[A1]], align 4
+!LLVM:   %[[V2:[0-9]+]] = add i32 %[[V0]], %[[V1]]
+!LLVM:   store i32 %[[V2]], ptr %[[A0]], align 4
+!LLVM:   ret void
+!LLVM: }
+
+
+program test
+
+call f(1, 2)
+
+contains
+
+subroutine f(x, y)
+  integer :: x, y
+  !$omp target data map(tofrom: x, y)
+  x = x + y
+  !$omp end target data
+end subroutine
+end
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 90ce06a0345c..eece8573f00e 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4378,6 +4378,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   llvm::OpenMPIRBuilder::TargetDataInfo info(/*RequiresDevicePointerInfo=*/true,
                                              /*SeparateBeginEndCalls=*/true);
+  bool isTargetDevice = ompBuilder->Config.isTargetDevice();
+  bool isOffloadEntry =
+      isTargetDevice || !ompBuilder->Config.TargetTriples.empty();
 
   LogicalResult result =
       llvm::TypeSwitch<Operation *, LogicalResult>(op)
@@ -4467,6 +4470,9 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
 
   if (failed(result))
     return failure();
+  // Pretend we have IF(false) if we're not doing offload.
+  if (!isOffloadEntry)
+    ifCond = builder.getFalse();
 
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
   MapInfoData mapData;
diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
index 971bea206854..e6ea3aaeec65 100644
--- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
@@ -1,15 +1,17 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
 
-llvm.func @_QPopenmp_target_data() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(99 : i32) : i32
-    llvm.store %3, %1 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(99 : i32) : i32
+      llvm.store %3, %1 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
@@ -38,23 +40,25 @@ llvm.func @_QPopenmp_target_data() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
-  %1 = llvm.mlir.constant(1023 : index) : i64
-  %2 = llvm.mlir.constant(0 : index) : i64
-  %3 = llvm.mlir.constant(1024 : index) : i64
-  %4 = llvm.mlir.constant(1 : index) : i64
-  %5 = omp.map.bounds   lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%3 : i64) stride(%4 : i64) start_idx(%4 : i64)
-  %6 = omp.map.info var_ptr(%0 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%5)  -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%6 : !llvm.ptr) {
-    %7 = llvm.mlir.constant(99 : i32) : i32
-    %8 = llvm.mlir.constant(1 : i64) : i64
-    %9 = llvm.mlir.constant(1 : i64) : i64
-    %10 = llvm.mlir.constant(0 : i64) : i64
-    %11 = llvm.getelementptr %0[0, %10] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<1024 x i32>
-    llvm.store %7, %11 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
+    %1 = llvm.mlir.constant(1023 : index) : i64
+    %2 = llvm.mlir.constant(0 : index) : i64
+    %3 = llvm.mlir.constant(1024 : index) : i64
+    %4 = llvm.mlir.constant(1 : index) : i64
+    %5 = omp.map.bounds   lower_bound(%2 : i64) upper_bound(%1 : i64) extent(%3 : i64) stride(%4 : i64) start_idx(%4 : i64)
+    %6 = omp.map.info var_ptr(%0 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%5)  -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%6 : !llvm.ptr) {
+      %7 = llvm.mlir.constant(99 : i32) : i32
+      %8 = llvm.mlir.constant(1 : i64) : i64
+      %9 = llvm.mlir.constant(1 : i64) : i64
+      %10 = llvm.mlir.constant(0 : i64) : i64
+      %11 = llvm.getelementptr %0[0, %10] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.array<1024 x i32>
+      llvm.store %7, %11 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4096]
@@ -85,50 +89,52 @@ llvm.func @_QPopenmp_target_data_region(%0 : !llvm.ptr) {
 
 // -----
 
-llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
-  %4 = llvm.mlir.constant(1 : i64) : i64
-  %5 = llvm.alloca %4 x i32 {bindc_name = "dvc", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEdvc"} : (i64) -> !llvm.ptr
-  %6 = llvm.mlir.constant(1 : i64) : i64
-  %7 = llvm.alloca %6 x i32 {bindc_name = "i", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEi"} : (i64) -> !llvm.ptr
-  %8 = llvm.mlir.constant(5 : i32) : i32
-  llvm.store %8, %7 : i32, !llvm.ptr
-  %9 = llvm.mlir.constant(2 : i32) : i32
-  llvm.store %9, %5 : i32, !llvm.ptr
-  %10 = llvm.load %7 : !llvm.ptr -> i32
-  %11 = llvm.mlir.constant(10 : i32) : i32
-  %12 = llvm.icmp "slt" %10, %11 : i32
-  %13 = llvm.load %5 : !llvm.ptr -> i32
-  %14 = llvm.mlir.constant(1023 : index) : i64
-  %15 = llvm.mlir.constant(0 : index) : i64
-  %16 = llvm.mlir.constant(1024 : index) : i64
-  %17 = llvm.mlir.constant(1 : index) : i64
-  %18 = omp.map.bounds   lower_bound(%15 : i64) upper_bound(%14 : i64) extent(%16 : i64) stride(%17 : i64) start_idx(%17 : i64)
-  %map1 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(to) capture(ByRef) bounds(%18) -> !llvm.ptr {name = ""}
-  %19 = llvm.mlir.constant(511 : index) : i64
-  %20 = llvm.mlir.constant(0 : index) : i64
-  %21 = llvm.mlir.constant(512 : index) : i64
-  %22 = llvm.mlir.constant(1 : index) : i64
-  %23 = omp.map.bounds   lower_bound(%20 : i64) upper_bound(%19 : i64) extent(%21 : i64) stride(%22 : i64) start_idx(%22 : i64)
-  %map2 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%23) -> !llvm.ptr {name = ""}
-  omp.target_enter_data   if(%12) device(%13 : i32) map_entries(%map1, %map2 : !llvm.ptr, !llvm.ptr)
-  %24 = llvm.load %7 : !llvm.ptr -> i32
-  %25 = llvm.mlir.constant(10 : i32) : i32
-  %26 = llvm.icmp "sgt" %24, %25 : i32
-  %27 = llvm.load %5 : !llvm.ptr -> i32
-  %28 = llvm.mlir.constant(1023 : index) : i64
-  %29 = llvm.mlir.constant(0 : index) : i64
-  %30 = llvm.mlir.constant(1024 : index) : i64
-  %31 = llvm.mlir.constant(1 : index) : i64
-  %32 = omp.map.bounds   lower_bound(%29 : i64) upper_bound(%28 : i64) extent(%30 : i64) stride(%31 : i64) start_idx(%31 : i64)
-  %map3 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%32) -> !llvm.ptr {name = ""}
-  %33 = llvm.mlir.constant(511 : index) : i64
-  %34 = llvm.mlir.constant(0 : index) : i64
-  %35 = llvm.mlir.constant(512 : index) : i64
-  %36 = llvm.mlir.constant(1 : index) : i64
-  %37 = omp.map.bounds   lower_bound(%34 : i64) upper_bound(%33 : i64) extent(%35 : i64) stride(%36 : i64) start_idx(%36 : i64)
-  %map4 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%37) -> !llvm.ptr {name = ""}
-  omp.target_exit_data   if(%26) device(%27 : i32) map_entries(%map3, %map4 : !llvm.ptr, !llvm.ptr)
-  llvm.return
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
+    %4 = llvm.mlir.constant(1 : i64) : i64
+    %5 = llvm.alloca %4 x i32 {bindc_name = "dvc", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEdvc"} : (i64) -> !llvm.ptr
+    %6 = llvm.mlir.constant(1 : i64) : i64
+    %7 = llvm.alloca %6 x i32 {bindc_name = "i", in_type = i32, operandSegmentSizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEi"} : (i64) -> !llvm.ptr
+    %8 = llvm.mlir.constant(5 : i32) : i32
+    llvm.store %8, %7 : i32, !llvm.ptr
+    %9 = llvm.mlir.constant(2 : i32) : i32
+    llvm.store %9, %5 : i32, !llvm.ptr
+    %10 = llvm.load %7 : !llvm.ptr -> i32
+    %11 = llvm.mlir.constant(10 : i32) : i32
+    %12 = llvm.icmp "slt" %10, %11 : i32
+    %13 = llvm.load %5 : !llvm.ptr -> i32
+    %14 = llvm.mlir.constant(1023 : index) : i64
+    %15 = llvm.mlir.constant(0 : index) : i64
+    %16 = llvm.mlir.constant(1024 : index) : i64
+    %17 = llvm.mlir.constant(1 : index) : i64
+    %18 = omp.map.bounds   lower_bound(%15 : i64) upper_bound(%14 : i64) extent(%16 : i64) stride(%17 : i64) start_idx(%17 : i64)
+    %map1 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(to) capture(ByRef) bounds(%18) -> !llvm.ptr {name = ""}
+    %19 = llvm.mlir.constant(511 : index) : i64
+    %20 = llvm.mlir.constant(0 : index) : i64
+    %21 = llvm.mlir.constant(512 : index) : i64
+    %22 = llvm.mlir.constant(1 : index) : i64
+    %23 = omp.map.bounds   lower_bound(%20 : i64) upper_bound(%19 : i64) extent(%21 : i64) stride(%22 : i64) start_idx(%22 : i64)
+    %map2 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%23) -> !llvm.ptr {name = ""}
+    omp.target_enter_data   if(%12) device(%13 : i32) map_entries(%map1, %map2 : !llvm.ptr, !llvm.ptr)
+    %24 = llvm.load %7 : !llvm.ptr -> i32
+    %25 = llvm.mlir.constant(10 : i32) : i32
+    %26 = llvm.icmp "sgt" %24, %25 : i32
+    %27 = llvm.load %5 : !llvm.ptr -> i32
+    %28 = llvm.mlir.constant(1023 : index) : i64
+    %29 = llvm.mlir.constant(0 : index) : i64
+    %30 = llvm.mlir.constant(1024 : index) : i64
+    %31 = llvm.mlir.constant(1 : index) : i64
+    %32 = omp.map.bounds   lower_bound(%29 : i64) upper_bound(%28 : i64) extent(%30 : i64) stride(%31 : i64) start_idx(%31 : i64)
+    %map3 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.array<1024 x i32>)   map_clauses(from) capture(ByRef) bounds(%32) -> !llvm.ptr {name = ""}
+    %33 = llvm.mlir.constant(511 : index) : i64
+    %34 = llvm.mlir.constant(0 : index) : i64
+    %35 = llvm.mlir.constant(512 : index) : i64
+    %36 = llvm.mlir.constant(1 : index) : i64
+    %37 = omp.map.bounds   lower_bound(%34 : i64) upper_bound(%33 : i64) extent(%35 : i64) stride(%36 : i64) start_idx(%36 : i64)
+    %map4 = omp.map.info var_ptr(%3 : !llvm.ptr, !llvm.array<512 x i32>)   map_clauses(exit_release_or_enter_alloc) capture(ByRef) bounds(%37) -> !llvm.ptr {name = ""}
+    omp.target_exit_data   if(%26) device(%27 : i32) map_entries(%map3, %map4 : !llvm.ptr, !llvm.ptr)
+    llvm.return
+  }
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 4096, i64 2048]
@@ -205,18 +211,20 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr, %3 : !llvm.ptr) {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_ptr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map1 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map1 : !llvm.ptr) use_device_ptr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %1, %2 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_ptr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map1 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map1 : !llvm.ptr) use_device_ptr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %1, %2 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
@@ -249,18 +257,20 @@ llvm.func @_QPopenmp_target_use_dev_ptr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %1, %2 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      %2 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %1, %2 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 8]
@@ -291,17 +301,19 @@ llvm.func @_QPopenmp_target_use_dev_addr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %1 = llvm.mlir.constant(10 : i32) : i32
-    llvm.store %1, %arg0 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x i32 : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, i32)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %1 = llvm.mlir.constant(10 : i32) : i32
+      llvm.store %1, %arg0 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
@@ -331,23 +343,25 @@ llvm.func @_QPopenmp_target_use_dev_addr_no_ptr() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %1 = llvm.mlir.constant(1 : i64) : i64
-  %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
-    %2 = llvm.mlir.constant(10 : i32) : i32
-    %3 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %2, %3 : i32, !llvm.ptr
-    %4 = llvm.mlir.constant(20 : i32) : i32
-    %5 = llvm.load %b : !llvm.ptr -> !llvm.ptr
-    llvm.store %4, %5 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %1 = llvm.mlir.constant(1 : i64) : i64
+    %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map : !llvm.ptr) use_device_addr(%map2 -> %arg0 : !llvm.ptr)  {
+      %2 = llvm.mlir.constant(10 : i32) : i32
+      %3 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %2, %3 : i32, !llvm.ptr
+      %4 = llvm.mlir.constant(20 : i32) : i32
+      %5 = llvm.load %b : !llvm.ptr -> !llvm.ptr
+      llvm.store %4, %5 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 8, i64 0]
@@ -387,25 +401,27 @@ llvm.func @_QPopenmp_target_use_dev_addr_nomap() {
 
 // -----
 
-llvm.func @_QPopenmp_target_use_dev_both() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %1 = llvm.mlir.constant(1 : i64) : i64
-  %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
-  %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map1 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  %map3 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data  map_entries(%map, %map1 : !llvm.ptr, !llvm.ptr) use_device_addr(%map3 -> %arg0 : !llvm.ptr) use_device_ptr(%map2 -> %arg1 : !llvm.ptr)  {
-    %2 = llvm.mlir.constant(10 : i32) : i32
-    %3 = llvm.load %arg1 : !llvm.ptr -> !llvm.ptr
-    llvm.store %2, %3 : i32, !llvm.ptr
-    %4 = llvm.mlir.constant(20 : i32) : i32
-    %5 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
-    llvm.store %4, %5 : i32, !llvm.ptr
-    omp.terminator
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_use_dev_both() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %a = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %1 = llvm.mlir.constant(1 : i64) : i64
+    %b = llvm.alloca %0 x !llvm.ptr : (i64) -> !llvm.ptr
+    %map = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map1 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map2 = omp.map.info var_ptr(%a : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    %map3 = omp.map.info var_ptr(%b : !llvm.ptr, !llvm.ptr)   map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data  map_entries(%map, %map1 : !llvm.ptr, !llvm.ptr) use_device_addr(%map3 -> %arg0 : !llvm.ptr) use_device_ptr(%map2 -> %arg1 : !llvm.ptr)  {
+      %2 = llvm.mlir.constant(10 : i32) : i32
+      %3 = llvm.load %arg1 : !llvm.ptr -> !llvm.ptr
+      llvm.store %2, %3 : i32, !llvm.ptr
+      %4 = llvm.mlir.constant(20 : i32) : i32
+      %5 = llvm.load %arg0 : !llvm.ptr -> !llvm.ptr
+      llvm.store %4, %5 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 8, i64 8]
@@ -448,19 +464,21 @@ llvm.func @_QPopenmp_target_use_dev_both() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_update() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(99 : i32) : i32
-    llvm.store %3, %1 : i32, !llvm.ptr
-    omp.terminator
-  }
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_update() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(99 : i32) : i32
+      llvm.store %3, %1 : i32, !llvm.ptr
+      omp.terminator
+    }
 
-  omp.target_update map_entries(%2 : !llvm.ptr)
+    omp.target_update map_entries(%2 : !llvm.ptr)
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK-LABEL: define void @_QPopenmp_target_data_update
@@ -488,26 +506,28 @@ llvm.func @_QPopenmp_target_data_update() {
 
 // -----
 
-omp.declare_mapper @_QQFmy_testmy_mapper : !llvm.struct<"_QFmy_testTmy_type", (i32)> {
-^bb0(%arg0: !llvm.ptr):
-  %0 = llvm.mlir.constant(0 : i32) : i32
-  %1 = llvm.getelementptr %arg0[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "var%data"}
-  %3 = omp.map.info var_ptr(%arg0 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) members(%2 : [0] : !llvm.ptr) -> !llvm.ptr {name = "var", partial_map = true}
-  omp.declare_mapper.info map_entries(%3, %2 : !llvm.ptr, !llvm.ptr)
-}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  omp.declare_mapper @_QQFmy_testmy_mapper : !llvm.struct<"_QFmy_testTmy_type", (i32)> {
+  ^bb0(%arg0: !llvm.ptr):
+    %0 = llvm.mlir.constant(0 : i32) : i32
+    %1 = llvm.getelementptr %arg0[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32) map_clauses(tofrom) capture(ByRef) -> !llvm.ptr {name = "var%data"}
+    %3 = omp.map.info var_ptr(%arg0 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) members(%2 : [0] : !llvm.ptr) -> !llvm.ptr {name = "var", partial_map = true}
+    omp.declare_mapper.info map_entries(%3, %2 : !llvm.ptr, !llvm.ptr)
+  }
 
-llvm.func @_QPopenmp_target_data_mapper() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x !llvm.struct<"_QFmy_testTmy_type", (i32)> {bindc_name = "a"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) mapper(@_QQFmy_testmy_mapper) -> !llvm.ptr {name = "a"}
-  omp.target_data map_entries(%2 : !llvm.ptr) {
-    %3 = llvm.mlir.constant(10 : i32) : i32
-    %4 = llvm.getelementptr %1[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
-    llvm.store %3, %4 : i32, !llvm.ptr
-    omp.terminator
+  llvm.func @_QPopenmp_target_data_mapper() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x !llvm.struct<"_QFmy_testTmy_type", (i32)> {bindc_name = "a"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>) map_clauses(tofrom) capture(ByRef) mapper(@_QQFmy_testmy_mapper) -> !llvm.ptr {name = "a"}
+    omp.target_data map_entries(%2 : !llvm.ptr) {
+      %3 = llvm.mlir.constant(10 : i32) : i32
+      %4 = llvm.getelementptr %1[0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"_QFmy_testTmy_type", (i32)>
+      llvm.store %3, %4 : i32, !llvm.ptr
+      omp.terminator
+    }
+    llvm.return
   }
-  llvm.return
 }
 
 // CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
diff --git a/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir b/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
index dba8c553aaca..f5c620a8942d 100644
--- a/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptargetdata-nowait-llvm.mlir
@@ -1,13 +1,15 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s 2>&1 | FileCheck %s
 
-llvm.func @_QPopenmp_target_data_enter() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_enter() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
+    omp.target_enter_data map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_enter() {
@@ -32,14 +34,16 @@ llvm.func @_QPopenmp_target_data_enter() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_update() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_update() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(to) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_update map_entries(%2 : !llvm.ptr) nowait
+    omp.target_update map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_update() {
@@ -64,14 +68,16 @@ llvm.func @_QPopenmp_target_data_update() {
 
 // -----
 
-llvm.func @_QPopenmp_target_data_exit() {
-  %0 = llvm.mlir.constant(1 : i64) : i64
-  %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
-  %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
+  llvm.func @_QPopenmp_target_data_exit() {
+    %0 = llvm.mlir.constant(1 : i64) : i64
+    %1 = llvm.alloca %0 x i32 {bindc_name = "i", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFopenmp_target_dataEi"} : (i64) -> !llvm.ptr
+    %2 = omp.map.info var_ptr(%1 : !llvm.ptr, i32)   map_clauses(from) capture(ByRef) -> !llvm.ptr {name = ""}
 
-  omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
+    omp.target_exit_data map_entries(%2 : !llvm.ptr) nowait
 
-  llvm.return
+    llvm.return
+  }
 }
 
 // CHECK: define void @_QPopenmp_target_data_exit() {
diff --git a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
index 717a77e61b9a..53c9b4f55964 100644
--- a/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-data-target-device.mlir
@@ -3,7 +3,7 @@
 // This tests checks that a target op inside a data op
 // We are only interested in ensuring that the -mlir-to-llmvir pass doesn't crash.
 // CHECK: {{.*}} = add i32 {{.*}}, 1
-module attributes { } {
+module attributes {omp.target_triples = ["amdgcn-amd-amdhsa"]} {
   llvm.mlir.global weak_odr hidden local_unnamed_addr constant @__oclc_ABI_version(400 : i32) {addr_space = 4 : i32} : i32
   llvm.func @_QQmain() attributes {fir.bindc_name = "main", omp.declare_target = #omp.declaretarget<device_type = (host), capture_clause = (to)>} {
     %0 = llvm.mlir.constant(99 : index) : i64
-- 
cgit v1.2.3


From 152d4b8a01e8671a676e7cfaf71c70b1edeee7e8 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Fri, 20 Jun 2025 14:23:26 +0100
Subject: [AArch64] Use indexed dup for 128b segmented splat (#144688)

Matches a splat of 128b segments into a wider z register expressed as a
concat_vectors sdnode and generate a dup zn.q, zd.q[0] instruction.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    |  10 ++
 .../AArch64/sve-fixed-length-splat-segment.ll      | 142 +++++++++++++++++++++
 2 files changed, 152 insertions(+)
 create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-splat-segment.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index eaaff0529cbd..e9d05710cbc4 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -29358,6 +29358,16 @@ SDValue AArch64TargetLowering::LowerFixedLengthConcatVectorsToSVE(
   EVT VT = Op.getValueType();
   EVT SrcVT = SrcOp1.getValueType();
 
+  // Match a splat of 128b segments that fit in a single register.
+  if (SrcVT.is128BitVector() && all_equal(Op.getNode()->op_values())) {
+    EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT);
+    SDValue Splat =
+        DAG.getNode(AArch64ISD::DUPLANE128, DL, ContainerVT,
+                    convertToScalableVector(DAG, ContainerVT, SrcOp1),
+                    DAG.getConstant(0, DL, MVT::i64, /*isTarget=*/true));
+    return convertFromScalableVector(DAG, VT, Splat);
+  }
+
   if (NumOperands > 2) {
     SmallVector<SDValue, 4> Ops;
     EVT PairVT = SrcVT.getDoubleNumVectorElementsVT(*DAG.getContext());
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-segment.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-segment.ll
new file mode 100644
index 000000000000..a5d213c658c6
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-splat-segment.ll
@@ -0,0 +1,142 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+;; Patterns that lower to concat_vectors where all incoming operands are the same.
+
+define void @concat_i8q_256(<16 x i8> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_i8q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <16 x i8> %data, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15,
+                                                                        i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+  store <32 x i8> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_i16q_256(<8 x i16> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_i16q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <8 x i16> %data, <8 x i16> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                        i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <16 x i16> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_i32q_256(<4 x i32> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_i32q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <4 x i32> %data, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                       i32 0, i32 1, i32 2, i32 3>
+  store <8 x i32> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_i64q_256(<2 x i64> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_i64q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <2 x i64> %data, <2 x i64> poison, <4 x i32> <i32 0, i32 1,
+                                                                       i32 0, i32 1>
+  store <4 x i64> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_f16q_256(<8 x half> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_f16q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <8 x half> %data, <8 x half> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                          i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <16 x half> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_bf16q_256(<8 x bfloat> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_bf16q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    stp q0, q0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <8 x bfloat> %data, <8 x bfloat> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
+                                                                              i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  store <16 x bfloat> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_f32q_256(<4 x float> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_f32q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <4 x float> %data, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                           i32 0, i32 1, i32 2, i32 3>
+  store <8 x float> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_f64q_256(<2 x double> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_f64q_256:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <2 x double> %data, <2 x double> poison, <4 x i32> <i32 0, i32 1,
+                                                                             i32 0, i32 1>
+  store <4 x double> %splat, ptr %addr, align 1
+  ret void
+}
+
+;; Test a wider vector
+
+define void @concat_i32q_512_with_256_vectors(<4 x i32> %data, ptr %addr) #0 {
+; CHECK-LABEL: concat_i32q_512_with_256_vectors:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0, #1, mul vl]
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <4 x i32> %data, <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3>
+  store <16 x i32> %splat, ptr %addr, align 1
+  ret void
+}
+
+define void @concat_i32q_512_with_512_vectors(<4 x i32> %data, ptr %addr) #1 {
+; CHECK-LABEL: concat_i32q_512_with_512_vectors:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT:    mov z0.q, q0
+; CHECK-NEXT:    str z0, [x0]
+; CHECK-NEXT:    ret
+  %splat = shufflevector <4 x i32> %data, <4 x i32> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3,
+                                                                        i32 0, i32 1, i32 2, i32 3>
+  store <16 x i32> %splat, ptr %addr, align 1
+  ret void
+}
+
+attributes #0 = { vscale_range(2,2) "target-features"="+sve,+bf16" }
+attributes #1 = { vscale_range(4,4) "target-features"="+sve,+bf16" }
-- 
cgit v1.2.3


From 8db272ffcf9ad97fe5614b34cd978eac30b53cd1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andrzej=20Warzy=C5=84ski?= <andrzej.warzynski@arm.com>
Date: Fri, 20 Jun 2025 14:25:36 +0100
Subject: [mlir][SparseTensor] Re-enable tests on AArch64 (#143387)

These tests were disabled in https://reviews.llvm.org/D136273, due to:
* https://github.com/llvm/llvm-project/issues/58465

That issue has now been resolved, so we should be able to re-enable
these tests.
---
 mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir | 2 --
 mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir   | 2 --
 2 files changed, 4 deletions(-)

diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
index bb7dcd2f35e3..40a0351a83ee 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dense_output_bf16.mlir
@@ -28,8 +28,6 @@
 // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true
 // RUN: %{compile} | %{run} | FileCheck %s
 
-// UNSUPPORTED: target=aarch64{{.*}}
-
 #SparseVector = #sparse_tensor.encoding<{map = (d0) -> (d0 : compressed)}>
 #DenseVector = #sparse_tensor.encoding<{map = (d0) -> (d0 : dense)}>
 
diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
index a0711d5b577a..440c1b63c8c2 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_sum_bf16.mlir
@@ -30,8 +30,6 @@
 // Do the same run, but now with  VLA vectorization.
 // RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %}
 
-// UNSUPPORTED: target=aarch64{{.*}}, mlir_arm_emulator
-
 !Filename = !llvm.ptr
 
 #SparseMatrix = #sparse_tensor.encoding<{
-- 
cgit v1.2.3


From c3929fbf0ef2d0fac05c54237bd9eac4fd57b1d8 Mon Sep 17 00:00:00 2001
From: Paul Walker <paul.walker@arm.com>
Date: Fri, 20 Jun 2025 13:16:52 +0000
Subject: [NFC][LLVM] Reorder SVE_SME codegen predicate classes.

Also removes unused HasNonStreamingSVE2p1_or_SSVE_AES predicate.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td | 45 +++++++++++++++++------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 400ffff5d567..efe6cc1aa8ae 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -245,7 +245,7 @@ def HasOCCMO         : Predicate<"Subtarget->hasOCCMO()">,
 // A subset of SVE(2) instructions are legal in Streaming SVE execution mode,
 // they should be enabled if either has been specified.
 def HasSVE_or_SME
-    : Predicate<"Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
+    : Predicate<"Subtarget->isSVEorStreamingSVEAvailable()">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
                 "sve or sme">;
 def HasNonStreamingSVE_or_SME2p2
@@ -253,6 +253,12 @@ def HasNonStreamingSVE_or_SME2p2
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME2p2),
                 "sve or sme2p2">;
+def HasNonStreamingSVE_or_SSVE_FEXPA
+    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
+                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
+                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA),
+                "sve or ssve-fexpa">;
+
 def HasSVE2_or_SME
     : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME),
@@ -264,40 +270,41 @@ def HasSVE2_or_SME2
 def HasNonStreamingSVE2_or_SSVE_AES
     : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_AES())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_AES), "sve2 or ssve-aes">;
+                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_AES),
+                "sve2 or ssve-aes">;
+
 def HasSVE2p1_or_SME
     : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
-                 AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1), "sme or sve2p1">;
+                AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1),
+                "sme or sve2p1">;
 def HasSVE2p1_or_SME2
     : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
-                 AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1), "sme2 or sve2p1">;
+                AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1),
+                "sme2 or sve2p1">;
 def HasSVE2p1_or_SME2p1
     : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
-                 AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1), "sme2p1 or sve2p1">;
+                AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1),
+                "sme2p1 or sve2p1">;
+
 def HasSVE2p2_or_SME2p2
     : Predicate<"Subtarget->isSVEorStreamingSVEAvailable() && (Subtarget->hasSVE2p2() || Subtarget->hasSME2p2())">,
-                 AssemblerPredicateWithAll<(any_of FeatureSME2p2, FeatureSVE2p2), "sme2p2 or sve2p2">;
-def HasNonStreamingSVE2p1_or_SSVE_AES
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2p1()) ||"
-                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_AES())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE2p1, FeatureSSVE_AES), "sve2p1 or ssve-aes">;
-def HasSMEF16F16_or_SMEF8F16
-    : Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
-                AssemblerPredicateWithAll<(any_of FeatureSMEF16F16, FeatureSMEF8F16),
-                "sme-f16f16 or sme-f8f16">;
+                AssemblerPredicateWithAll<(any_of FeatureSME2p2, FeatureSVE2p2),
+                "sme2p2 or sve2p2">;
 def HasNonStreamingSVE2p2_or_SME2p2
     : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2p2()) ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSME2p2())">,
                 AssemblerPredicateWithAll<(any_of FeatureSVE2p2, FeatureSME2p2),
                 "sme2p2 or sve2p2">;
+
+def HasSMEF16F16_or_SMEF8F16
+    : Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
+                AssemblerPredicateWithAll<(any_of FeatureSMEF16F16, FeatureSMEF8F16),
+                "sme-f16f16 or sme-f8f16">;
 def HasNonStreamingSVE2_or_SSVE_BitPerm
     : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE2()) ||"
                 "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_BitPerm())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_BitPerm), "sve2 or ssve-bitperm">;
-def HasNonStreamingSVE_or_SSVE_FEXPA
-    : Predicate<"(Subtarget->isSVEAvailable() && Subtarget->hasSVE()) ||"
-                "(Subtarget->isSVEorStreamingSVEAvailable() && Subtarget->hasSSVE_FEXPA())">,
-                AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSSVE_FEXPA), "sve or ssve-fexpa">;
+                AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSSVE_BitPerm),
+                "sve2 or ssve-bitperm">;
 
 // A subset of NEON instructions are legal in Streaming SVE execution mode,
 // so don't need the additional check for 'isNeonAvailable'.
-- 
cgit v1.2.3


From 376b71442d03bcc8ec6e2244002e3d62916dcea4 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi@nvidia.com>
Date: Fri, 20 Jun 2025 06:41:48 -0700
Subject: [NFC][TableGen][DecoderEmitter] Use structured binding in range for
 loop (#144890)

Also assign variable names to different elements of `OpMap` for better
readibility, and eliminate `NumberedEncodingsRef` as `std::vector` will
automatically get converted to an `ArrayRef`.
---
 llvm/utils/TableGen/DecoderEmitter.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/llvm/utils/TableGen/DecoderEmitter.cpp b/llvm/utils/TableGen/DecoderEmitter.cpp
index 37814113b467..2e8ff2aa47d9 100644
--- a/llvm/utils/TableGen/DecoderEmitter.cpp
+++ b/llvm/utils/TableGen/DecoderEmitter.cpp
@@ -2631,12 +2631,12 @@ namespace {
 
   DecoderTableInfo TableInfo;
   unsigned OpcodeMask = 0;
-  for (const auto &Opc : OpcMap) {
+  for (const auto &[NSAndByteSize, EncodingIDs] : OpcMap) {
+    const std::string &DecoderNamespace = NSAndByteSize.first;
+    const unsigned BitWidth = 8 * NSAndByteSize.second;
     // Emit the decoder for this namespace+width combination.
-    ArrayRef<EncodingAndInst> NumberedEncodingsRef(NumberedEncodings.data(),
-                                                   NumberedEncodings.size());
-    FilterChooser FC(NumberedEncodingsRef, Opc.second, Operands,
-                     IsVarLenInst ? MaxInstLen : 8 * Opc.first.second, this);
+    FilterChooser FC(NumberedEncodings, EncodingIDs, Operands,
+                     IsVarLenInst ? MaxInstLen : BitWidth, this);
 
     // The decode table is cleared for each top level decoder function. The
     // predicates and decoders themselves, however, are shared across all
@@ -2657,7 +2657,7 @@ namespace {
 
     // Print the table to the output stream.
     OpcodeMask |= emitTable(OS, TableInfo.Table, indent(0), FC.getBitWidth(),
-                            Opc.first.first, Opc.second);
+                            DecoderNamespace, EncodingIDs);
   }
 
   // For variable instruction, we emit a instruction length table
-- 
cgit v1.2.3


From 269cb22ae82fd83ecc7a7ef7f7a4110e4c7a43ec Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <Nico.Vasilache@amd.com>
Date: Fri, 20 Jun 2025 15:45:21 +0200
Subject: =?UTF-8?q?[mlir][transform]=20extract=20a=20minimal=20DomainAndOp?=
 =?UTF-8?q?erandsAffineMapT=E2=80=A6=20(#145034)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…ransferInterface out of LinalgStructuredInterface and use that for
PadTilingInterface

Along the way, a bug was found on the handling of scalar values, fix it
and add a test.
---
 .../mlir/Dialect/Linalg/IR/LinalgInterfaces.td     | 88 +++++++++++++---------
 .../mlir/Dialect/Linalg/Transforms/Transforms.h    |  9 +--
 .../Linalg/TransformOps/LinalgTransformOps.cpp     | 10 ++-
 .../Linalg/Transforms/PadTilingInterface.cpp       | 31 ++++++--
 .../Linalg/transform-op-pad-tiling-interface.mlir  | 28 +++++++
 5 files changed, 112 insertions(+), 54 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
index dbc1ac60e097..74c4c0a8835f 100644
--- a/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
+++ b/mlir/include/mlir/Dialect/Linalg/IR/LinalgInterfaces.td
@@ -222,9 +222,59 @@ def LinalgFillOpInterface : OpInterface<"FillOpInterface"> {
   ];
 }
 
+def IndexingMapOpInterface : OpInterface<"IndexingMapOpInterface"> {
+  let description = [{
+    Interface for operations that connect an iteration domain to operands via
+    affine maps. Provides methods to access indexing maps between iteration
+    domain and operand index spaces.
+  }];
+  let cppNamespace = "::mlir::linalg";
+  let methods = [
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the indexing maps attribute within the current operation.
+      }],
+      /*retTy=*/"ArrayAttr",
+      /*methodName=*/"getIndexingMaps"
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the indexing maps within the current operation.
+      }],
+      /*retTy=*/"SmallVector<AffineMap>",
+      /*methodName=*/"getIndexingMapsArray",
+      /*args=*/(ins),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        auto range = $_op.getIndexingMaps()
+          .template getAsValueRange<AffineMapAttr>();
+        return {range.begin(), range.end()};
+      }]
+    >,
+    InterfaceMethod<
+      /*desc=*/[{
+        Return the input or output indexing map for `opOperand`.
+      }],
+      /*retTy=*/"AffineMap",
+      /*methodName=*/"getMatchingIndexingMap",
+      /*args=*/(ins "OpOperand*":$opOperand),
+      /*methodBody=*/"",
+      /*defaultImplementation=*/[{
+        assert(opOperand->getOwner() == this->getOperation());
+        auto indexingMaps =
+          $_op.getIndexingMaps().template getAsValueRange<AffineMapAttr>();
+        return *(indexingMaps.begin() + opOperand->getOperandNumber());
+      }]
+    >,
+  ];
+}
+
 // The 'LinalgStructuredInterface' provides access to the 'LinalgOp' interface.
 def LinalgStructuredInterface
-    : OpInterface<"LinalgOp", [DestinationStyleOpInterface]> {
+    : OpInterface<"LinalgOp", [
+      DestinationStyleOpInterface,
+      IndexingMapOpInterface
+  ]> {
   let cppNamespace = "::mlir::linalg";
   let methods = [
     //===------------------------------------------------------------------===//
@@ -465,21 +515,6 @@ def LinalgStructuredInterface
             blockArgument.getArgNumber());
       }]
     >,
-    InterfaceMethod<
-      /*desc=*/[{
-        Return the input or output indexing map for `opOperand`.
-      }],
-      /*retTy=*/"AffineMap",
-      /*methodName=*/"getMatchingIndexingMap",
-      /*args=*/(ins "OpOperand*":$opOperand),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{
-        assert(opOperand->getOwner() == this->getOperation());
-        auto indexingMaps =
-          $_op.getIndexingMaps().template getAsValueRange<AffineMapAttr>();
-        return *(indexingMaps.begin() + opOperand->getOperandNumber());
-      }]
-    >,
     InterfaceMethod<
       /*desc=*/[{
         Return the indexing map for a `result`.
@@ -576,27 +611,6 @@ def LinalgStructuredInterface
       /*methodBody=*/"",
       /*defaultImplementation=*/[{ return success(); }]
     >,
-    InterfaceMethod<
-      /*desc=*/[{
-        Return the indexing maps attribute within the current operation.
-      }],
-      /*retTy=*/"ArrayAttr",
-      /*methodName=*/"getIndexingMaps"
-    >,
-    InterfaceMethod<
-      /*desc=*/[{
-        Return the indexing maps within the current operation.
-      }],
-      /*retTy=*/"SmallVector<AffineMap>",
-      /*methodName=*/"getIndexingMapsArray",
-      /*args=*/(ins),
-      /*methodBody=*/"",
-      /*defaultImplementation=*/[{
-        auto range = $_op.getIndexingMaps()
-          .template getAsValueRange<AffineMapAttr>();
-        return {range.begin(), range.end()};
-      }]
-    >,
     InterfaceMethod<
       /*desc=*/[{
         Return true if any of the operands has a dynamic shape.
diff --git a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
index 59b7fdeef10b..a6dab03d6473 100644
--- a/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/Linalg/Transforms/Transforms.h
@@ -612,10 +612,9 @@ using PadSizeComputationFunction =
         const PadTilingInterfaceOptions &)>;
 
 /// Specific helper for Linalg ops.
-FailureOr<SmallVector<OpFoldResult>>
-computeLinalgPaddedShape(RewriterBase &rewriter, OpOperand &operandToPad,
-                         ArrayRef<Range> iterationDomain,
-                         const PadTilingInterfaceOptions &options);
+FailureOr<SmallVector<OpFoldResult>> computeIndexingMapOpInterfacePaddedShape(
+    RewriterBase &rewriter, OpOperand &operandToPad,
+    ArrayRef<Range> iterationDomain, const PadTilingInterfaceOptions &options);
 
 /// Pad the iterator dimensions `options.paddingDimensions` of `opToPad`.
 ///
@@ -632,7 +631,7 @@ rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
                   const PadTilingInterfaceOptions &constOptions,
                   SmallVector<tensor::PadOp> &padOps,
                   PadSizeComputationFunction computePaddingSizeFun =
-                      &computeLinalgPaddedShape);
+                      &computeIndexingMapOpInterfacePaddedShape);
 
 namespace detail {
 
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index e627fc83f2ba..5d55adbf46f3 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -2229,10 +2229,12 @@ transform::PadTilingInterfaceOp::apply(transform::TransformRewriter &rewriter,
       return diag;
     }
 
-    // Only Linalg ops for now, until TilingInterface exposes a loopsToOperand
-    // map / C++ APIs to compute the effect of padding on operands.
-    if (!isa<LinalgOp>(targetOp.getOperation())) {
-      auto diag = emitSilenceableError() << "only LinalgOp supported atm";
+    // Only IndexingMapOpInterface ops for now, until TilingInterface exposes a
+    // loopsToOperand map / C++ APIs to compute the effect of padding on
+    // operands.
+    if (!isa<IndexingMapOpInterface>(targetOp.getOperation())) {
+      auto diag = emitSilenceableError() << "only IndexingMapOpInterface ops "
+                                            "supported atm";
       diag.attachNote(target->getLoc()) << "target op";
       return diag;
     }
diff --git a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
index a9d7bc64f2a6..5383ae48aeb3 100644
--- a/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
+++ b/mlir/lib/Dialect/Linalg/Transforms/PadTilingInterface.cpp
@@ -155,11 +155,13 @@ SmallVector<OpFoldResult> linalg::computePaddedShape(
   return paddedShape;
 }
 
-FailureOr<SmallVector<OpFoldResult>> linalg::computeLinalgPaddedShape(
+FailureOr<SmallVector<OpFoldResult>>
+linalg::computeIndexingMapOpInterfacePaddedShape(
     RewriterBase &rewriter, OpOperand &operandToPad,
     ArrayRef<Range> iterationDomain, const PadTilingInterfaceOptions &options) {
-  auto linalgOp = llvm::dyn_cast<LinalgOp>(operandToPad.getOwner());
-  if (!linalgOp)
+  auto transferOp =
+      llvm::dyn_cast<IndexingMapOpInterface>(operandToPad.getOwner());
+  if (!transferOp)
     return failure();
 
   // clang-format off
@@ -173,7 +175,7 @@ FailureOr<SmallVector<OpFoldResult>> linalg::computeLinalgPaddedShape(
   for (const Range &range : iterationDomain)
     loopUpperBounds.push_back(range.size);
 
-  AffineMap indexingMap = linalgOp.getMatchingIndexingMap(&operandToPad);
+  AffineMap indexingMap = transferOp.getMatchingIndexingMap(&operandToPad);
   return computePaddedShape(
       rewriter, cast<TypedValue<RankedTensorType>>(operandToPad.get()),
       indexingMap, loopUpperBounds, options);
@@ -255,7 +257,18 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
   SmallVector<Value> newOperands;
   newOperands.reserve(opToPad->getNumOperands());
   for (OpOperand &opOperand : opToPad->getOpOperands()) {
-    LLVM_DEBUG(DBGS() << "--start padding oprd: " << opOperand.get() << "\n");
+    Value operand = opOperand.get();
+    LLVM_DEBUG(DBGS() << "--start padding oprd: " << operand << "\n");
+
+    // 2.a. Skip scalar-like operands.
+    Type operandType = operand.getType();
+    if (!isa<RankedTensorType>(operandType)) {
+      assert(!isa<ShapedType>(operandType) ||
+             isa<VectorType>(operandType) &&
+                 "Unexpected non-vector ShapedType");
+      newOperands.push_back(operand);
+      continue;
+    }
     // 2.a. Compute padded shape.
     FailureOr<SmallVector<OpFoldResult>> maybePaddedShape =
         computePaddingSizeFun(rewriter, opOperand, iterationDomain, options);
@@ -266,14 +279,16 @@ linalg::rewriteAsPaddedOp(RewriterBase &rewriter, TilingInterface opToPad,
     // 2.b. Expect proper `paddingValues`.
     // TODO: we may want to allow garbage padding in the future, in which case
     // we would just not assert.
-    assert(opOperand.getOperandNumber() < options.paddingValues.size() &&
-           "--no padding value specified");
+    if (opOperand.getOperandNumber() >= options.paddingValues.size()) {
+      return rewriter.notifyMatchFailure(opToPad,
+                                         "--no padding value specified");
+    }
     Attribute paddingValueAttr =
         options.paddingValues[opOperand.getOperandNumber()];
 
     // 2.c. Perform actual padding.
     Value paddedOperand = padOperand(
-        rewriter, opToPad, cast<TypedValue<RankedTensorType>>(opOperand.get()),
+        rewriter, opToPad, cast<TypedValue<RankedTensorType>>(operand),
         *maybePaddedShape, paddingValueAttr);
     LLVM_DEBUG(DBGS() << "--done padding operand: " << paddedOperand << "\n");
 
diff --git a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
index c361885693cb..f0a410fa4015 100644
--- a/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
+++ b/mlir/test/Dialect/Linalg/transform-op-pad-tiling-interface.mlir
@@ -1,5 +1,33 @@
 // RUN: mlir-opt --transform-interpreter -canonicalize -split-input-file --verify-diagnostics %s | FileCheck %s
 
+//     CHECK-LABEL: pad_fill
+//           CHECK:   linalg.fill ins(%{{.*}} : f32) outs(%{{.*}} : tensor<8x25xf32>) -> tensor<8x25xf32>
+func.func @pad_fill(%value: f32, %output: tensor<24x25xf32>) -> tensor<24x25xf32>
+{
+  %0 = linalg.fill ins(%value : f32) outs(%output : tensor<24x25xf32>) -> tensor<24x25xf32>
+  func.return %0 : tensor<24x25xf32>
+}
+
+module attributes {transform.with_named_sequence} {
+  transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) {
+    %fill = transform.structured.match ops{["linalg.fill"]} in %arg1
+      : (!transform.any_op) -> !transform.any_op
+
+    // Tile to 5 then pad to 8
+    %fill_l1, %loops_l1 = transform.structured.tile_using_for %fill tile_sizes [5] 
+      : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+    %fill_padded, %_ = transform.structured.pad_tiling_interface %fill_l1 to padding_sizes [8] {
+      padding_values=[0.0 : f32, 0.0 : f32],
+      padding_dimensions=[0]
+    } : (!transform.any_op) -> (!transform.any_op, !transform.any_op)
+
+    transform.yield
+  }
+}
+
+// -----
+
 //     CHECK-LABEL: pad_lhs
 func.func @pad_lhs(
   %arg0: tensor<24x12xf32>, %arg1: tensor<12x25xf32>, %arg2: tensor<24x25xf32>)
-- 
cgit v1.2.3


From 3ff69c80786e4c2e37fd40a48ee675e33a5b6f76 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hunter@arm.com>
Date: Fri, 20 Jun 2025 14:49:50 +0100
Subject: [LV] Add early-exit-with-store tests (#140899)

Adds some additional LoopVectorizeLegality tests for early exit loops
with a store that we don't vectorize.

Test precommit split from #137774
---
 .../LoopVectorize/early_exit_legality.ll           |  45 +--
 .../LoopVectorize/early_exit_store_legality.ll     | 409 +++++++++++++++++++++
 2 files changed, 429 insertions(+), 25 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll

diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
index de455c81d363..22b34079755c 100644
--- a/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_legality.ll
@@ -442,35 +442,30 @@ loop.end:
   ret i64 %retval
 }
 
-
-define i64 @loop_contains_store(ptr %dest) {
-; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
-; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops
+define void @exit_conditions_combined_in_single_branch(ptr noalias dereferenceable(40) %array, ptr readonly align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'exit_conditions_combined_in_single_branch'
+; CHECK:       LV: Not vectorizing: Cannot vectorize uncountable loop.
 entry:
-  %p1 = alloca [1024 x i8]
-  call void @init_mem(ptr %p1, i64 1024)
-  br label %loop
-
-loop:
-  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
-  %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index
-  %ld1 = load i32, ptr %arrayidx, align 1
-  %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
-  store i32 %ld1, ptr %arrayidx2, align 4
-  %cmp = icmp eq i32 %ld1, 1
-  br i1 %cmp, label %loop.inc, label %loop.end
-
-loop.inc:
-  %index.next = add i64 %index, 1
-  %exitcond = icmp ne i64 %index.next, 67
-  br i1 %exitcond, label %loop, label %loop.end
+  br label %for.body
 
-loop.end:
-  %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
-  ret i64 %retval
+for.body:                                         ; preds = %for.body, %entry
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  %or.cond = select i1 %ee.cond, i1 true, i1 %counted.cond
+  br i1 %or.cond, label %exit, label %for.body
+
+exit:                                             ; preds = %for.body
+  ret void
 }
 
-
 define i64 @uncountable_exit_in_conditional_block(ptr %mask) {
 ; CHECK-LABEL: LV: Checking a loop in 'uncountable_exit_in_conditional_block'
 ; CHECK:       LV: Not vectorizing: Early exit is not the latch predecessor.
diff --git a/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
new file mode 100644
index 000000000000..3a08681a2fb9
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/early_exit_store_legality.ll
@@ -0,0 +1,409 @@
+; REQUIRES: asserts
+; RUN: opt -S < %s -p loop-vectorize -debug-only=loop-vectorize -enable-early-exit-vectorization -force-vector-width=4 -disable-output 2>&1 | FileCheck %s
+
+define i64 @loop_contains_store(ptr %dest) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops
+entry:
+  %p1 = alloca [1024 x i8]
+  call void @init_mem(ptr %p1, i64 1024)
+  br label %loop
+
+loop:
+  %index = phi i64 [ %index.next, %loop.inc ], [ 3, %entry ]
+  %arrayidx = getelementptr inbounds i32, ptr %p1, i64 %index
+  %ld1 = load i32, ptr %arrayidx, align 1
+  %arrayidx2 = getelementptr inbounds i32, ptr %dest, i64 %index
+  store i32 %ld1, ptr %arrayidx2, align 4
+  %cmp = icmp eq i32 %ld1, 1
+  br i1 %cmp, label %loop.inc, label %loop.end
+
+loop.inc:
+  %index.next = add i64 %index, 1
+  %exitcond = icmp ne i64 %index.next, 67
+  br i1 %exitcond, label %loop, label %loop.end
+
+loop.end:
+  %retval = phi i64 [ %index, %loop ], [ 67, %loop.inc ]
+  ret i64 %retval
+}
+
+define void @loop_contains_store_condition_load_has_single_user(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_has_single_user'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_ee_condition_is_invariant(ptr dereferenceable(40) noalias %array, i16 %ee.val) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_ee_condition_is_invariant'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_fcmp_condition(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_fcmp_condition'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw half, ptr %pred, i64 %iv
+  %ee.val = load half, ptr %ee.addr, align 2
+  %ee.cond = fcmp ugt half %ee.val, 500.0
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_safe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(96) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_safe_dependency'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  %pred.plus.8 = getelementptr inbounds nuw i16, ptr %pred, i64 8
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred.plus.8, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  %some.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  store i16 42, ptr %some.addr, align 2
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_unsafe_dependency(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(80) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unsafe_dependency'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  %unknown.offset = call i64 @get_an_unknown_offset()
+  %unknown.cmp = icmp ult i64 %unknown.offset, 20
+  %clamped.offset = select i1 %unknown.cmp, i64 %unknown.offset, i64 20
+  %unknown.base = getelementptr i16, ptr %pred, i64 %clamped.offset
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  %some.addr = getelementptr inbounds nuw i16, ptr %unknown.base, i64 %iv
+  store i16 42, ptr %some.addr, align 2
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_assumed_bounds(ptr noalias %array, ptr readonly %pred, i32 %n) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_assumed_bounds'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  %n_bytes = mul nuw nsw i32 %n, 2
+  call void @llvm.assume(i1 true) [ "align"(ptr %pred, i64 2), "dereferenceable"(ptr %pred, i32 %n_bytes) ]
+  %tc = sext i32 %n to i64
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, %tc
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_to_pointer_with_no_deref_info(ptr align 2 dereferenceable(40) readonly %load.array, ptr align 2 noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_pointer_with_no_deref_info'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %ld.addr = getelementptr inbounds nuw i16, ptr %load.array, i64 %iv
+  %data = load i16, ptr %ld.addr, align 2
+  %inc = add nsw i16 %data, 1
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_unknown_bounds(ptr align 2 dereferenceable(100) noalias %array, ptr align 2 dereferenceable(100) readonly %pred, i64 %n) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_unknown_bounds'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, %n
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_volatile(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_volatile'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store volatile i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_to_invariant_location(ptr dereferenceable(40) readonly %array, ptr align 2 dereferenceable(40) readonly %pred, ptr noalias %store_addr) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_to_invariant_location'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %store_addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_in_latch_block(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_in_latch_block'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  store i16 %inc, ptr %st.addr, align 2
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_requiring_alias_check(ptr dereferenceable(40) %array, ptr align 2 dereferenceable(40) %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_requiring_alias_check'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_condition_load_is_chained(ptr dereferenceable(40) noalias %array, ptr align 8 dereferenceable(160) readonly %offsets, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_condition_load_is_chained'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 0, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %gather.addr = getelementptr inbounds nuw i64, ptr %offsets, i64 %iv
+  %ee.offset = load i64, ptr %gather.addr, align 8
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %ee.offset
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = add nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 20
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+define void @loop_contains_store_decrementing_iv(ptr dereferenceable(40) noalias %array, ptr align 2 dereferenceable(40) readonly %pred) {
+; CHECK-LABEL: LV: Checking a loop in 'loop_contains_store_decrementing_iv'
+; CHECK:       LV: Not vectorizing: Writes to memory unsupported in early exit loops.
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i64 [ 19, %entry ], [ %iv.next, %for.inc ]
+  %st.addr = getelementptr inbounds nuw i16, ptr %array, i64 %iv
+  %data = load i16, ptr %st.addr, align 2
+  %inc = add nsw i16 %data, 1
+  store i16 %inc, ptr %st.addr, align 2
+  %ee.addr = getelementptr inbounds nuw i16, ptr %pred, i64 %iv
+  %ee.val = load i16, ptr %ee.addr, align 2
+  %ee.cond = icmp sgt i16 %ee.val, 500
+  br i1 %ee.cond, label %exit, label %for.inc
+
+for.inc:
+  %iv.next = sub nuw nsw i64 %iv, 1
+  %counted.cond = icmp eq i64 %iv.next, 0
+  br i1 %counted.cond, label %exit, label %for.body
+
+exit:
+  ret void
+}
+
+declare void @init_mem(ptr, i64);
+declare i64 @get_an_unknown_offset();
-- 
cgit v1.2.3


From 225768d1f9f2e2ccff7dc79b4a4aaeab4c6aafc1 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 14:51:01 +0100
Subject: [X86] combineConcatVectorOps - add tests showing v4i64 shift-by-32
 with unnecessary concatenation

On AVX1-only targets, we concat SHL/SRL AVX1 v4i64 by 32-bits as a shuffle. But this is only worth while if the shift source value is free to concatenate.
---
 llvm/test/CodeGen/X86/vector-shift-lshr-256.ll | 67 ++++++++++++++++++++++++++
 llvm/test/CodeGen/X86/vector-shift-shl-256.ll  | 67 ++++++++++++++++++++++++++
 2 files changed, 134 insertions(+)

diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index d8852956c66f..b45525b6e20f 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -1971,6 +1971,73 @@ define <4 x i64> @shift32_v4i64(<4 x i64> %a) nounwind {
   ret <4 x i64> %shift
 }
 
+define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
+; AVX1-LABEL: shift32_v4i64_concat:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shift32_v4i64_concat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: shift32_v4i64_concat:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
+; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: shift32_v4i64_concat:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512-LABEL: shift32_v4i64_concat:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; AVX512VL-LABEL: shift32_v4i64_concat:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; X86-AVX1-LABEL: shift32_v4i64_concat:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
+; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: shift32_v4i64_concat:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsrlq $32, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
+  %a = shufflevector <2 x i64> %lo, <2 x i64> %hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shift = lshr <4 x i64> %a, <i64 32, i64 32, i64 32, i64 32>
+  ret <4 x i64> %shift
+}
+
 define <4 x i32> @sh_trunc_sh_vec(<4 x i64> %x) {
 ; AVX1-LABEL: sh_trunc_sh_vec:
 ; AVX1:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 3f238b5739f0..2248ee997d52 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -1823,3 +1823,70 @@ define <4 x i64> @shift32_v4i64(<4 x i64> %a) nounwind {
   %shift = shl <4 x i64> %a, <i64 32, i64 32, i64 32, i64 32>
   ret <4 x i64> %shift
 }
+
+define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
+; AVX1-LABEL: shift32_v4i64_concat:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
+; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: shift32_v4i64_concat:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX2-NEXT:    retq
+;
+; XOPAVX1-LABEL: shift32_v4i64_concat:
+; XOPAVX1:       # %bb.0:
+; XOPAVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
+; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; XOPAVX1-NEXT:    retq
+;
+; XOPAVX2-LABEL: shift32_v4i64_concat:
+; XOPAVX2:       # %bb.0:
+; XOPAVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; XOPAVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
+; XOPAVX2-NEXT:    retq
+;
+; AVX512-LABEL: shift32_v4i64_concat:
+; AVX512:       # %bb.0:
+; AVX512-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512-NEXT:    retq
+;
+; AVX512VL-LABEL: shift32_v4i64_concat:
+; AVX512VL:       # %bb.0:
+; AVX512VL-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX512VL-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT:    vpsllq $32, %ymm0, %ymm0
+; AVX512VL-NEXT:    retq
+;
+; X86-AVX1-LABEL: shift32_v4i64_concat:
+; X86-AVX1:       # %bb.0:
+; X86-AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
+; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; X86-AVX1-NEXT:    retl
+;
+; X86-AVX2-LABEL: shift32_v4i64_concat:
+; X86-AVX2:       # %bb.0:
+; X86-AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT:    vpsllq $32, %ymm0, %ymm0
+; X86-AVX2-NEXT:    retl
+  %a = shufflevector <2 x i64> %lo, <2 x i64> %hi, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %shift = shl <4 x i64> %a, <i64 32, i64 32, i64 32, i64 32>
+  ret <4 x i64> %shift
+}
-- 
cgit v1.2.3


From bd36f7331a9f575272aebb9e0163194541110912 Mon Sep 17 00:00:00 2001
From: Andres-Salamanca <andrealebarbaritos@gmail.com>
Date: Fri, 20 Jun 2025 09:03:02 -0500
Subject: [CIR] Add initial support for bitfields in structs (#142041)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This change adds support for bitfields CIR records can now contain bit
fields.

I’ve updated the `CIRGenBitFieldInfo` comment, which originally came
from the incubator and was identical to the one in OGCodeGen, to better
reflect the current implementation.

Support for bitfields in unions big-endian architectures and `get` and
`set` operations remains unimplemented and will be addressed in a future
patch.
---
 clang/include/clang/CIR/MissingFeatures.h          |   3 +
 clang/lib/CIR/CodeGen/CIRGenRecordLayout.h         | 114 +++++++++
 .../lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp  | 266 ++++++++++++++++++++-
 clang/lib/CIR/CodeGen/TargetInfo.cpp               |  37 +++
 clang/lib/CIR/CodeGen/TargetInfo.h                 |  10 +
 clang/test/CIR/CodeGen/bitfields.c                 |  78 ++++++
 clang/test/CIR/CodeGen/bitfields.cpp               |  32 +++
 7 files changed, 534 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/CIR/CodeGen/bitfields.c
 create mode 100644 clang/test/CIR/CodeGen/bitfields.cpp

diff --git a/clang/include/clang/CIR/MissingFeatures.h b/clang/include/clang/CIR/MissingFeatures.h
index 45452c5929a3..e0b2959f374f 100644
--- a/clang/include/clang/CIR/MissingFeatures.h
+++ b/clang/include/clang/CIR/MissingFeatures.h
@@ -149,6 +149,8 @@ struct MissingFeatures {
   static bool cxxabiUseARMGuardVarABI() { return false; }
   static bool cxxabiAppleARM64CXXABI() { return false; }
   static bool cxxabiStructorImplicitParam() { return false; }
+  static bool isDiscreteBitFieldABI() { return false; }
+  static bool isBigEndian() { return false; }
 
   // Address class
   static bool addressOffset() { return false; }
@@ -239,6 +241,7 @@ struct MissingFeatures {
   static bool builtinCall() { return false; }
   static bool builtinCallF128() { return false; }
   static bool builtinCallMathErrno() { return false; }
+  static bool nonFineGrainedBitfields() { return false; }
 
   // Missing types
   static bool dataMemberType() { return false; }
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h b/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
index ac8832b8c9b2..3b51ab784d37 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayout.h
@@ -14,6 +14,106 @@
 
 namespace clang::CIRGen {
 
+/// Record with information about how a bitfield should be accessed. This is
+/// very similar to what LLVM codegen does, once CIR evolves it's possible we
+/// can use a more higher level representation.
+///
+/// Often we lay out a sequence of bitfields as a contiguous sequence of bits.
+/// When the AST record layout does this, we represent it in CIR as a
+/// `!cir.record` type, which directly reflects the structure's layout,
+/// including bitfield packing and padding, using CIR types such as
+/// `!cir.bool`, `!s8i`, `!u16i`.
+///
+/// To access a particular bitfield in CIR, we use the operations
+/// `cir.get_bitfield` (`GetBitfieldOp`) or `cir.set_bitfield`
+/// (`SetBitfieldOp`). These operations rely on the `bitfield_info`
+/// attribute, which provides detailed metadata required for access,
+/// such as the size and offset of the bitfield, the type and size of
+/// the underlying storage, and whether the value is signed.
+/// The CIRGenRecordLayout also has a bitFields map which encodes which
+/// byte-sequence this bitfield falls within. Let's assume the following C
+/// struct:
+///
+///   struct S {
+///     char a, b, c;
+///     unsigned bits : 3;
+///     unsigned more_bits : 4;
+///     unsigned still_more_bits : 7;
+///   };
+///
+/// This will end up as the following cir.record. The bitfield members are
+/// represented by one !u16i value, and the array provides padding to align the
+/// struct to a 4-byte alignment.
+///
+///   !rec_S = !cir.record<struct "S" padded {!s8i, !s8i, !s8i, !u16i,
+///   !cir.array<!u8i x 3>}>
+///
+/// When generating code to access more_bits, we'll generate something
+/// essentially like this:
+///
+///   #bfi_more_bits = #cir.bitfield_info<name = "more_bits", storage_type =
+///   !u16i, size = 4, offset = 3, is_signed = false>
+///
+///   cir.func @store_field() {
+///     %0 = cir.alloca !rec_S, !cir.ptr<!rec_S>, ["s"] {alignment = 4 : i64}
+///     %1 = cir.const #cir.int<2> : !s32i
+///     %2 = cir.cast(integral, %1 : !s32i), !u32i
+///     %3 = cir.get_member %0[3] {name = "more_bits"} : !cir.ptr<!rec_S> ->
+///     !cir.ptr<!u16i>
+///     %4 = cir.set_bitfield(#bfi_more_bits, %3 :
+///     !cir.ptr<!u16i>, %2 : !u32i) -> !u32i
+///     cir.return
+///   }
+///
+struct CIRGenBitFieldInfo {
+  /// The offset within a contiguous run of bitfields that are represented as
+  /// a single "field" within the cir.record type. This offset is in bits.
+  unsigned offset : 16;
+
+  /// The total size of the bit-field, in bits.
+  unsigned size : 15;
+
+  /// Whether the bit-field is signed.
+  unsigned isSigned : 1;
+
+  /// The storage size in bits which should be used when accessing this
+  /// bitfield.
+  unsigned storageSize;
+
+  /// The offset of the bitfield storage from the start of the record.
+  clang::CharUnits storageOffset;
+
+  /// The offset within a contiguous run of bitfields that are represented as a
+  /// single "field" within the cir.record type, taking into account the AAPCS
+  /// rules for volatile bitfields. This offset is in bits.
+  unsigned volatileOffset : 16;
+
+  /// The storage size in bits which should be used when accessing this
+  /// bitfield.
+  unsigned volatileStorageSize;
+
+  /// The offset of the bitfield storage from the start of the record.
+  clang::CharUnits volatileStorageOffset;
+
+  /// The name of a bitfield
+  llvm::StringRef name;
+
+  // The actual storage type for the bitfield
+  mlir::Type storageType;
+
+  CIRGenBitFieldInfo()
+      : offset(), size(), isSigned(), storageSize(), volatileOffset(),
+        volatileStorageSize() {}
+
+  CIRGenBitFieldInfo(unsigned offset, unsigned size, bool isSigned,
+                     unsigned storageSize, clang::CharUnits storageOffset)
+      : offset(offset), size(size), isSigned(isSigned),
+        storageSize(storageSize), storageOffset(storageOffset) {}
+
+  void print(llvm::raw_ostream &os) const;
+  LLVM_DUMP_METHOD void dump() const;
+};
+
 /// This class handles record and union layout info while lowering AST types
 /// to CIR types.
 ///
@@ -41,6 +141,10 @@ private:
   // for both virtual and non-virtual bases.
   llvm::DenseMap<const clang::CXXRecordDecl *, unsigned> nonVirtualBases;
 
+  /// Map from (bit-field) record field to the corresponding CIR record type
+  /// field no. This info is populated by record builder.
+  llvm::DenseMap<const clang::FieldDecl *, CIRGenBitFieldInfo> bitFields;
+
   /// False if any direct or indirect subobject of this class, when considered
   /// as a complete object, requires a non-zero bitpattern when
   /// zero-initialized.
@@ -83,6 +187,16 @@ public:
   /// Check whether this struct can be C++ zero-initialized
   /// with a zeroinitializer when considered as a base subobject.
   bool isZeroInitializableAsBase() const { return zeroInitializableAsBase; }
+
+  /// Return the BitFieldInfo that corresponds to the field FD.
+  const CIRGenBitFieldInfo &getBitFieldInfo(const clang::FieldDecl *fd) const {
+    fd = fd->getCanonicalDecl();
+    assert(fd->isBitField() && "Invalid call for non-bit-field decl!");
+    llvm::DenseMap<const clang::FieldDecl *, CIRGenBitFieldInfo>::const_iterator
+        it = bitFields.find(fd);
+    assert(it != bitFields.end() && "Unable to find bitfield info");
+    return it->second;
+  }
 };
 
 } // namespace clang::CIRGen
diff --git a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
index 0aeef7fd89ae..8dbf1b36a93b 100644
--- a/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenRecordLayoutBuilder.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/RecordLayout.h"
 #include "clang/CIR/Dialect/IR/CIRAttrs.h"
 #include "clang/CIR/Dialect/IR/CIRDataLayout.h"
+#include "clang/CIR/MissingFeatures.h"
 #include "llvm/Support/Casting.h"
 
 #include <memory>
@@ -66,6 +67,10 @@ struct CIRRecordLowering final {
     return MemberInfo(offset, MemberInfo::InfoKind::Field, data);
   }
 
+  // Layout routines.
+  void setBitFieldInfo(const FieldDecl *fd, CharUnits startOffset,
+                       mlir::Type storageType);
+
   void lower();
   void lowerUnion();
 
@@ -77,6 +82,9 @@ struct CIRRecordLowering final {
   void accumulateBases(const CXXRecordDecl *cxxRecordDecl);
   void accumulateVPtrs();
   void accumulateFields();
+  RecordDecl::field_iterator
+  accumulateBitFields(RecordDecl::field_iterator field,
+                      RecordDecl::field_iterator fieldEnd);
 
   CharUnits bitsToCharUnits(uint64_t bitOffset) {
     return astContext.toCharUnitsFromBits(bitOffset);
@@ -87,6 +95,9 @@ struct CIRRecordLowering final {
   CharUnits getSize(mlir::Type Ty) {
     return CharUnits::fromQuantity(dataLayout.layout.getTypeSize(Ty));
   }
+  CharUnits getSizeInBits(mlir::Type ty) {
+    return CharUnits::fromQuantity(dataLayout.layout.getTypeSizeInBits(ty));
+  }
   CharUnits getAlignment(mlir::Type Ty) {
     return CharUnits::fromQuantity(dataLayout.layout.getTypeABIAlignment(Ty));
   }
@@ -124,6 +135,17 @@ struct CIRRecordLowering final {
   mlir::Type getStorageType(const CXXRecordDecl *RD) {
     return cirGenTypes.getCIRGenRecordLayout(RD).getBaseSubobjectCIRType();
   }
+  // This is different from LLVM traditional codegen because CIRGen uses arrays
+  // of bytes instead of arbitrary-sized integers. This is important for packed
+  // structures support.
+  mlir::Type getBitfieldStorageType(unsigned numBits) {
+    unsigned alignedBits = llvm::alignTo(numBits, astContext.getCharWidth());
+    if (cir::isValidFundamentalIntWidth(alignedBits))
+      return builder.getUIntNTy(alignedBits);
+
+    mlir::Type type = getCharType();
+    return cir::ArrayType::get(type, alignedBits / astContext.getCharWidth());
+  }
 
   mlir::Type getStorageType(const FieldDecl *fieldDecl) {
     mlir::Type type = cirGenTypes.convertTypeForMem(fieldDecl->getType());
@@ -157,6 +179,7 @@ struct CIRRecordLowering final {
   std::vector<MemberInfo> members;
   // Output fields, consumed by CIRGenTypes::computeRecordLayout
   llvm::SmallVector<mlir::Type, 16> fieldTypes;
+  llvm::DenseMap<const FieldDecl *, CIRGenBitFieldInfo> bitFields;
   llvm::DenseMap<const FieldDecl *, unsigned> fieldIdxMap;
   llvm::DenseMap<const CXXRecordDecl *, unsigned> nonVirtualBases;
   cir::CIRDataLayout dataLayout;
@@ -186,6 +209,32 @@ CIRRecordLowering::CIRRecordLowering(CIRGenTypes &cirGenTypes,
       zeroInitializable(true), zeroInitializableAsBase(true), packed(packed),
       padded(false) {}
 
+void CIRRecordLowering::setBitFieldInfo(const FieldDecl *fd,
+                                        CharUnits startOffset,
+                                        mlir::Type storageType) {
+  CIRGenBitFieldInfo &info = bitFields[fd->getCanonicalDecl()];
+  info.isSigned = fd->getType()->isSignedIntegerOrEnumerationType();
+  info.offset =
+      (unsigned)(getFieldBitOffset(fd) - astContext.toBits(startOffset));
+  info.size = fd->getBitWidthValue();
+  info.storageSize = getSizeInBits(storageType).getQuantity();
+  info.storageOffset = startOffset;
+  info.storageType = storageType;
+  info.name = fd->getName();
+
+  if (info.size > info.storageSize)
+    info.size = info.storageSize;
+  // Reverse the bit offsets for big endian machines. Since bitfields are laid
+  // out as packed bits within an integer-sized unit, we can imagine the bits
+  // counting from the most-significant-bit instead of the
+  // least-significant-bit.
+  assert(!cir::MissingFeatures::isBigEndian());
+
+  info.volatileStorageSize = 0;
+  info.volatileOffset = 0;
+  info.volatileStorageOffset = CharUnits::Zero();
+}
+
 void CIRRecordLowering::lower() {
   if (recordDecl->isUnion()) {
     lowerUnion();
@@ -233,6 +282,8 @@ void CIRRecordLowering::fillOutputFields() {
             fieldTypes.size() - 1;
       // A field without storage must be a bitfield.
       assert(!cir::MissingFeatures::bitfields());
+      if (!member.data)
+        setBitFieldInfo(member.fieldDecl, member.offset, fieldTypes.back());
     } else if (member.kind == MemberInfo::InfoKind::Base) {
       nonVirtualBases[member.cxxRecordDecl] = fieldTypes.size() - 1;
     }
@@ -240,16 +291,217 @@ void CIRRecordLowering::fillOutputFields() {
   }
 }
 
+RecordDecl::field_iterator
+CIRRecordLowering::accumulateBitFields(RecordDecl::field_iterator field,
+                                       RecordDecl::field_iterator fieldEnd) {
+  assert(!cir::MissingFeatures::isDiscreteBitFieldABI());
+
+  CharUnits regSize =
+      bitsToCharUnits(astContext.getTargetInfo().getRegisterWidth());
+  unsigned charBits = astContext.getCharWidth();
+
+  // Data about the start of the span we're accumulating to create an access
+  // unit from. 'Begin' is the first bitfield of the span. If 'begin' is
+  // 'fieldEnd', we've not got a current span. The span starts at the
+  // 'beginOffset' character boundary. 'bitSizeSinceBegin' is the size (in bits)
+  // of the span -- this might include padding when we've advanced to a
+  // subsequent bitfield run.
+  RecordDecl::field_iterator begin = fieldEnd;
+  CharUnits beginOffset;
+  uint64_t bitSizeSinceBegin;
+
+  // The (non-inclusive) end of the largest acceptable access unit we've found
+  // since 'begin'. If this is 'begin', we're gathering the initial set of
+  // bitfields of a new span. 'bestEndOffset' is the end of that acceptable
+  // access unit -- it might extend beyond the last character of the bitfield
+  // run, using available padding characters.
+  RecordDecl::field_iterator bestEnd = begin;
+  CharUnits bestEndOffset;
+  bool bestClipped; // Whether the representation must be in a byte array.
+
+  for (;;) {
+    // atAlignedBoundary is true if 'field' is the (potential) start of a new
+    // span (or the end of the bitfields). When true, limitOffset is the
+    // character offset of that span and barrier indicates whether the new
+    // span cannot be merged into the current one.
+    bool atAlignedBoundary = false;
+    bool barrier = false; // a barrier can be a zero Bit Width or non bit member
+    if (field != fieldEnd && field->isBitField()) {
+      uint64_t bitOffset = getFieldBitOffset(*field);
+      if (begin == fieldEnd) {
+        // Beginning a new span.
+        begin = field;
+        bestEnd = begin;
+
+        assert((bitOffset % charBits) == 0 && "Not at start of char");
+        beginOffset = bitsToCharUnits(bitOffset);
+        bitSizeSinceBegin = 0;
+      } else if ((bitOffset % charBits) != 0) {
+        // Bitfield occupies the same character as previous bitfield, it must be
+        // part of the same span. This can include zero-length bitfields, should
+        // the target not align them to character boundaries. Such non-alignment
+        // is at variance with the standards, which require zero-length
+        // bitfields be a barrier between access units. But of course we can't
+        // achieve that in the middle of a character.
+        assert(bitOffset ==
+                   astContext.toBits(beginOffset) + bitSizeSinceBegin &&
+               "Concatenating non-contiguous bitfields");
+      } else {
+        // Bitfield potentially begins a new span. This includes zero-length
+        // bitfields on non-aligning targets that lie at character boundaries
+        // (those are barriers to merging).
+        if (field->isZeroLengthBitField())
+          barrier = true;
+        atAlignedBoundary = true;
+      }
+    } else {
+      // We've reached the end of the bitfield run. Either we're done, or this
+      // is a barrier for the current span.
+      if (begin == fieldEnd)
+        break;
+
+      barrier = true;
+      atAlignedBoundary = true;
+    }
+
+    // 'installBest' indicates whether we should create an access unit for the
+    // current best span: fields ['begin', 'bestEnd') occupying characters
+    // ['beginOffset', 'bestEndOffset').
+    bool installBest = false;
+    if (atAlignedBoundary) {
+      // 'field' is the start of a new span or the end of the bitfields. The
+      // just-seen span now extends to 'bitSizeSinceBegin'.
+
+      // Determine if we can accumulate that just-seen span into the current
+      // accumulation.
+      CharUnits accessSize = bitsToCharUnits(bitSizeSinceBegin + charBits - 1);
+      if (bestEnd == begin) {
+        // This is the initial run at the start of a new span. By definition,
+        // this is the best seen so far.
+        bestEnd = field;
+        bestEndOffset = beginOffset + accessSize;
+        // Assume clipped until proven not below.
+        bestClipped = true;
+        if (!bitSizeSinceBegin)
+          // A zero-sized initial span -- this will install nothing and reset
+          // for another.
+          installBest = true;
+      } else if (accessSize > regSize) {
+        // Accumulating the just-seen span would create a multi-register access
+        // unit, which would increase register pressure.
+        installBest = true;
+      }
+
+      if (!installBest) {
+        // Determine if accumulating the just-seen span will create an expensive
+        // access unit or not.
+        mlir::Type type = getUIntNType(astContext.toBits(accessSize));
+        if (!astContext.getTargetInfo().hasCheapUnalignedBitFieldAccess())
+          cirGenTypes.getCGModule().errorNYI(
+              field->getSourceRange(), "NYI CheapUnalignedBitFieldAccess");
+
+        if (!installBest) {
+          // Find the next used storage offset to determine what the limit of
+          // the current span is. That's either the offset of the next field
+          // with storage (which might be field itself) or the end of the
+          // non-reusable tail padding.
+          CharUnits limitOffset;
+          for (auto probe = field; probe != fieldEnd; ++probe)
+            if (!isEmptyFieldForLayout(astContext, *probe)) {
+              // A member with storage sets the limit.
+              assert((getFieldBitOffset(*probe) % charBits) == 0 &&
+                     "Next storage is not byte-aligned");
+              limitOffset = bitsToCharUnits(getFieldBitOffset(*probe));
+              goto FoundLimit;
+            }
+          assert(!cir::MissingFeatures::cxxSupport());
+          limitOffset = astRecordLayout.getDataSize();
+        FoundLimit:
+          CharUnits typeSize = getSize(type);
+          if (beginOffset + typeSize <= limitOffset) {
+            // There is space before limitOffset to create a naturally-sized
+            // access unit.
+            bestEndOffset = beginOffset + typeSize;
+            bestEnd = field;
+            bestClipped = false;
+          }
+          if (barrier) {
+            // The next field is a barrier that we cannot merge across.
+            installBest = true;
+          } else if (cirGenTypes.getCGModule()
+                         .getCodeGenOpts()
+                         .FineGrainedBitfieldAccesses) {
+            assert(!cir::MissingFeatures::nonFineGrainedBitfields());
+            cirGenTypes.getCGModule().errorNYI(field->getSourceRange(),
+                                               "NYI FineGrainedBitfield");
+          } else {
+            // Otherwise, we're not installing. Update the bit size
+            // of the current span to go all the way to limitOffset, which is
+            // the (aligned) offset of next bitfield to consider.
+            bitSizeSinceBegin = astContext.toBits(limitOffset - beginOffset);
+          }
+        }
+      }
+    }
+
+    if (installBest) {
+      assert((field == fieldEnd || !field->isBitField() ||
+              (getFieldBitOffset(*field) % charBits) == 0) &&
+             "Installing but not at an aligned bitfield or limit");
+      CharUnits accessSize = bestEndOffset - beginOffset;
+      if (!accessSize.isZero()) {
+        // Add the storage member for the access unit to the record. The
+        // bitfields get the offset of their storage but come afterward and
+        // remain there after a stable sort.
+        mlir::Type type;
+        if (bestClipped) {
+          assert(getSize(getUIntNType(astContext.toBits(accessSize))) >
+                     accessSize &&
+                 "Clipped access need not be clipped");
+          type = getByteArrayType(accessSize);
+        } else {
+          type = getUIntNType(astContext.toBits(accessSize));
+          assert(getSize(type) == accessSize &&
+                 "Unclipped access must be clipped");
+        }
+        members.push_back(makeStorageInfo(beginOffset, type));
+        for (; begin != bestEnd; ++begin)
+          if (!begin->isZeroLengthBitField())
+            members.push_back(MemberInfo(
+                beginOffset, MemberInfo::InfoKind::Field, nullptr, *begin));
+      }
+      // Reset to start a new span.
+      field = bestEnd;
+      begin = fieldEnd;
+    } else {
+      assert(field != fieldEnd && field->isBitField() &&
+             "Accumulating past end of bitfields");
+      assert(!barrier && "Accumulating across barrier");
+      // Accumulate this bitfield into the current (potential) span.
+      bitSizeSinceBegin += field->getBitWidthValue();
+      ++field;
+    }
+  }
+
+  return field;
+}
+
 void CIRRecordLowering::accumulateFields() {
-  for (const FieldDecl *field : recordDecl->fields()) {
+  for (RecordDecl::field_iterator field = recordDecl->field_begin(),
+                                  fieldEnd = recordDecl->field_end();
+       field != fieldEnd;) {
     if (field->isBitField()) {
-      cirGenTypes.getCGModule().errorNYI(recordDecl->getSourceRange(),
-                                         "accumulate bitfields");
-      ++field;
+      RecordDecl::field_iterator start = field;
+      // Iterate to gather the list of bitfields.
+      for (++field; field != fieldEnd && field->isBitField(); ++field)
+        ;
+      field = accumulateBitFields(start, field);
+      assert((field == fieldEnd || !field->isBitField()) &&
+             "Failed to accumulate all the bitfields");
     } else if (!field->isZeroSize(astContext)) {
-      members.push_back(MemberInfo(bitsToCharUnits(getFieldBitOffset(field)),
+      members.push_back(MemberInfo(bitsToCharUnits(getFieldBitOffset(*field)),
                                    MemberInfo::InfoKind::Field,
-                                   getStorageType(field), field));
+                                   getStorageType(*field), *field));
       ++field;
     } else {
       // TODO(cir): do we want to do anything special about zero size members?
@@ -383,6 +635,8 @@ CIRGenTypes::computeRecordLayout(const RecordDecl *rd, cir::RecordType *ty) {
   // Add all the field numbers.
   rl->fieldIdxMap.swap(lowering.fieldIdxMap);
 
+  rl->bitFields.swap(lowering.bitFields);
+
   // Dump the layout, if requested.
   if (getASTContext().getLangOpts().DumpRecordLayouts) {
     cgm.errorNYI(rd->getSourceRange(), "computeRecordLayout: dump layout");
diff --git a/clang/lib/CIR/CodeGen/TargetInfo.cpp b/clang/lib/CIR/CodeGen/TargetInfo.cpp
index 551341ff20c0..d2d32bbd9403 100644
--- a/clang/lib/CIR/CodeGen/TargetInfo.cpp
+++ b/clang/lib/CIR/CodeGen/TargetInfo.cpp
@@ -3,6 +3,43 @@
 
 using namespace clang;
 using namespace clang::CIRGen;
+
+bool clang::CIRGen::isEmptyRecordForLayout(const ASTContext &context,
+                                           QualType t) {
+  const RecordType *rt = t->getAs<RecordType>();
+  if (!rt)
+    return false;
+
+  const RecordDecl *rd = rt->getDecl();
+
+  // If this is a C++ record, check the bases first.
+  if (const CXXRecordDecl *cxxrd = dyn_cast<CXXRecordDecl>(rd)) {
+    if (cxxrd->isDynamicClass())
+      return false;
+
+    for (const auto &I : cxxrd->bases())
+      if (!isEmptyRecordForLayout(context, I.getType()))
+        return false;
+  }
+
+  for (const auto *I : rd->fields())
+    if (!isEmptyFieldForLayout(context, I))
+      return false;
+
+  return true;
+}
+
+bool clang::CIRGen::isEmptyFieldForLayout(const ASTContext &context,
+                                          const FieldDecl *fd) {
+  if (fd->isZeroLengthBitField())
+    return true;
+
+  if (fd->isUnnamedBitField())
+    return false;
+
+  return isEmptyRecordForLayout(context, fd->getType());
+}
+
 namespace {
 
 class X8664ABIInfo : public ABIInfo {
diff --git a/clang/lib/CIR/CodeGen/TargetInfo.h b/clang/lib/CIR/CodeGen/TargetInfo.h
index d31d1ee82d90..a5c548aa2c7c 100644
--- a/clang/lib/CIR/CodeGen/TargetInfo.h
+++ b/clang/lib/CIR/CodeGen/TargetInfo.h
@@ -22,6 +22,16 @@
 
 namespace clang::CIRGen {
 
+/// isEmptyFieldForLayout - Return true if the field is "empty", that is,
+/// either a zero-width bit-field or an isEmptyRecordForLayout.
+bool isEmptyFieldForLayout(const ASTContext &context, const FieldDecl *fd);
+
+/// isEmptyRecordForLayout - Return true if a structure contains only empty
+/// base classes (per  isEmptyRecordForLayout) and fields (per
+/// isEmptyFieldForLayout). Note, C++ record fields are considered empty
+/// if the [[no_unique_address]] attribute would have made them empty.
+bool isEmptyRecordForLayout(const ASTContext &context, QualType t);
+
 class TargetCIRGenInfo {
   std::unique_ptr<ABIInfo> info;
 
diff --git a/clang/test/CIR/CodeGen/bitfields.c b/clang/test/CIR/CodeGen/bitfields.c
new file mode 100644
index 000000000000..ff5c6bc1787b
--- /dev/null
+++ b/clang/test/CIR/CodeGen/bitfields.c
@@ -0,0 +1,78 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+typedef struct {
+  char a, b, c;
+  unsigned bits : 3;
+  unsigned more_bits : 4;
+  unsigned still_more_bits : 7;
+} A;
+
+// CIR-DAG:  !rec_A = !cir.record<struct "A" packed padded {!s8i, !s8i, !s8i, !u16i, !cir.array<!u8i x 3>}>
+// LLVM-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
+// OGCG-DAG: %struct.A = type <{ i8, i8, i8, i16, [3 x i8] }>
+
+typedef struct {
+  int a : 4;
+  int b : 5;
+  int c;
+} D;
+
+// CIR-DAG:  !rec_D = !cir.record<struct "D" {!u16i, !s32i}>
+// LLVM-DAG: %struct.D = type { i16, i32 }
+// OGCG-DAG: %struct.D = type { i16, i32 }
+
+typedef struct {
+  int a : 4;
+  int b : 27;
+  int c : 17;
+  int d : 2;
+  int e : 15;
+  unsigned f; // type other than int above, not a bitfield
+} S;
+// CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
+// LLVM-DAG: %struct.S = type { i64, i16, i32 }
+// OGCG-DAG: %struct.S = type { i64, i16, i32 }
+
+typedef struct {
+  int a : 3;  // one bitfield with size < 8
+  unsigned b;
+} T;
+
+// CIR-DAG:  !rec_T = !cir.record<struct "T" {!u8i, !u32i}>
+// LLVM-DAG: %struct.T = type { i8, i32 }
+// OGCG-DAG: %struct.T = type { i8, i32 }
+
+typedef struct {
+    char a;
+    char b;
+    char c;
+
+    // startOffset 24 bits, new storage from here
+    int d: 2;
+    int e: 2;
+    int f: 4;
+    int g: 25;
+    int h: 3;
+    int i: 4;
+    int j: 3;
+    int k: 8;
+
+    int l: 14;
+} U;
+
+// CIR-DAG:  !rec_U = !cir.record<struct "U" packed {!s8i, !s8i, !s8i, !u8i, !u64i}>
+// LLVM-DAG: %struct.U = type <{ i8, i8, i8, i8, i64 }>
+// OGCG-DAG: %struct.U = type <{ i8, i8, i8, i8, i64 }>
+
+void def() {
+  A a;
+  D d;
+  S s;
+  T t;
+  U u;
+}
diff --git a/clang/test/CIR/CodeGen/bitfields.cpp b/clang/test/CIR/CodeGen/bitfields.cpp
new file mode 100644
index 000000000000..762d24988474
--- /dev/null
+++ b/clang/test/CIR/CodeGen/bitfields.cpp
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --input-file=%t.cir %s --check-prefix=CIR
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t-cir.ll
+// RUN: FileCheck --input-file=%t-cir.ll %s --check-prefix=LLVM
+// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-linux-gnu -emit-llvm %s -o %t.ll
+// RUN: FileCheck --input-file=%t.ll %s --check-prefix=OGCG
+
+typedef struct {
+  int a : 4;
+  int b : 27;
+  int c : 17;
+  int d : 2;
+  int e : 15;
+  unsigned f; // type other than int above, not a bitfield
+} S;
+// CIR-DAG:  !rec_S = !cir.record<struct "S" {!u64i, !u16i, !u32i}>
+// LLVM-DAG: %struct.S = type { i64, i16, i32 }
+// OGCG-DAG: %struct.S = type { i64, i16, i32 }
+
+typedef struct {
+  int a : 3;  // one bitfield with size < 8
+  unsigned b;
+} T;
+
+// CIR-DAG:  !rec_T = !cir.record<struct "T" {!u8i, !u32i}>
+// LLVM-DAG: %struct.T = type { i8, i32 }
+// OGCG-DAG: %struct.T = type { i8, i32 }
+
+void def() {
+  S s;
+  T t;
+}
-- 
cgit v1.2.3


From e0633d59b9d0f931d3917e51a51b5aa7d7e775ac Mon Sep 17 00:00:00 2001
From: Ross Brunton <ross@codeplay.com>
Date: Fri, 20 Jun 2025 15:04:50 +0100
Subject: [Offload] Check for initialization (#144370)

All entry points (except olInit) now check that offload has been
initialized. If not, a new `OL_ERRC_UNINITIALIZED` error is returned.
---
 offload/liboffload/API/Common.td                   |  1 +
 offload/liboffload/include/OffloadImpl.hpp         |  1 +
 offload/liboffload/src/OffloadImpl.cpp             |  3 ++-
 offload/tools/offload-tblgen/EntryPointGen.cpp     | 12 ++++++++++++
 offload/unittests/OffloadAPI/CMakeLists.txt        |  4 ++++
 .../unittests/OffloadAPI/common/Environment.cpp    |  2 ++
 offload/unittests/OffloadAPI/init/olInit.cpp       | 22 ++++++++++++++++++++++
 7 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 offload/unittests/OffloadAPI/init/olInit.cpp

diff --git a/offload/liboffload/API/Common.td b/offload/liboffload/API/Common.td
index 8a2ecd6c6e8f..cd8c3c63fde8 100644
--- a/offload/liboffload/API/Common.td
+++ b/offload/liboffload/API/Common.td
@@ -106,6 +106,7 @@ def ErrorCode : Enum {
     Etor<"ASSEMBLE_FAILURE", "assembler failure while processing binary image">,
     Etor<"LINK_FAILURE", "linker failure while processing binary image">,
     Etor<"BACKEND_FAILURE", "the plugin backend is in an invalid or unsupported state">,
+    Etor<"UNINITIALIZED", "not initialized">,
 
     // Handle related errors - only makes sense for liboffload
     Etor<"INVALID_NULL_HANDLE", "a handle argument is null when it should not be">,
diff --git a/offload/liboffload/include/OffloadImpl.hpp b/offload/liboffload/include/OffloadImpl.hpp
index a12d8c47a180..f98164d5e178 100644
--- a/offload/liboffload/include/OffloadImpl.hpp
+++ b/offload/liboffload/include/OffloadImpl.hpp
@@ -26,6 +26,7 @@ namespace llvm {
 namespace offload {
 bool isTracingEnabled();
 bool isValidationEnabled();
+bool isOffloadInitialized();
 } // namespace offload
 } // namespace llvm
 
diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index f02497c0a633..8a487563f610 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -120,9 +120,10 @@ struct OffloadContext {
 
 // If the context is uninited, then we assume tracing is disabled
 bool isTracingEnabled() {
-  return OffloadContextVal && OffloadContext::get().TracingEnabled;
+  return isOffloadInitialized() && OffloadContext::get().TracingEnabled;
 }
 bool isValidationEnabled() { return OffloadContext::get().ValidationEnabled; }
+bool isOffloadInitialized() { return OffloadContextVal != nullptr; }
 
 template <typename HandleT> Error olDestroy(HandleT Handle) {
   delete Handle;
diff --git a/offload/tools/offload-tblgen/EntryPointGen.cpp b/offload/tools/offload-tblgen/EntryPointGen.cpp
index 13aa0d1f6318..4e42e4905b99 100644
--- a/offload/tools/offload-tblgen/EntryPointGen.cpp
+++ b/offload/tools/offload-tblgen/EntryPointGen.cpp
@@ -82,6 +82,10 @@ static void EmitEntryPointFunc(const FunctionRec &F, raw_ostream &OS) {
   }
   OS << ") {\n";
 
+  // Check offload is initialized
+  if (F.getName() != "olInit")
+    OS << "if (!llvm::offload::isOffloadInitialized()) return &UninitError;";
+
   // Emit pre-call prints
   OS << TAB_1 "if (llvm::offload::isTracingEnabled()) {\n";
   OS << formatv(TAB_2 "llvm::errs() << \"---> {0}\";\n", F.getName());
@@ -143,6 +147,14 @@ static void EmitCodeLocWrapper(const FunctionRec &F, raw_ostream &OS) {
 
 void EmitOffloadEntryPoints(const RecordKeeper &Records, raw_ostream &OS) {
   OS << GenericHeader;
+
+  constexpr const char *UninitMessage =
+      "liboffload has not been initialized - please call olInit before using "
+      "this API";
+  OS << formatv("static {0}_error_struct_t UninitError = "
+                "{{{1}_ERRC_UNINITIALIZED, \"{2}\"};",
+                PrefixLower, PrefixUpper, UninitMessage);
+
   for (auto *R : Records.getAllDerivedDefinitions("Function")) {
     EmitValidationFunc(FunctionRec{R}, OS);
     EmitEntryPointFunc(FunctionRec{R}, OS);
diff --git a/offload/unittests/OffloadAPI/CMakeLists.txt b/offload/unittests/OffloadAPI/CMakeLists.txt
index 2844b675e5de..05e862865ed3 100644
--- a/offload/unittests/OffloadAPI/CMakeLists.txt
+++ b/offload/unittests/OffloadAPI/CMakeLists.txt
@@ -12,6 +12,10 @@ add_offload_unittest("event"
     event/olDestroyEvent.cpp
     event/olWaitEvent.cpp)
 
+add_offload_unittest("init"
+    init/olInit.cpp)
+target_compile_definitions("init.unittests" PRIVATE DISABLE_WRAPPER)
+
 add_offload_unittest("kernel"
     kernel/olGetKernel.cpp
     kernel/olLaunchKernel.cpp)
diff --git a/offload/unittests/OffloadAPI/common/Environment.cpp b/offload/unittests/OffloadAPI/common/Environment.cpp
index 943347246b6d..ef092cd4187d 100644
--- a/offload/unittests/OffloadAPI/common/Environment.cpp
+++ b/offload/unittests/OffloadAPI/common/Environment.cpp
@@ -17,11 +17,13 @@ using namespace llvm;
 
 // Wrapper so we don't have to constantly init and shutdown Offload in every
 // test, while having sensible lifetime for the platform environment
+#ifndef DISABLE_WRAPPER
 struct OffloadInitWrapper {
   OffloadInitWrapper() { olInit(); }
   ~OffloadInitWrapper() { olShutDown(); }
 };
 static OffloadInitWrapper Wrapper{};
+#endif
 
 static cl::opt<std::string>
     SelectedPlatform("platform", cl::desc("Only test the specified platform"),
diff --git a/offload/unittests/OffloadAPI/init/olInit.cpp b/offload/unittests/OffloadAPI/init/olInit.cpp
new file mode 100644
index 000000000000..8e27e77cd0fb
--- /dev/null
+++ b/offload/unittests/OffloadAPI/init/olInit.cpp
@@ -0,0 +1,22 @@
+//===------- Offload API tests - olInit -----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// NOTE: For this test suite, the implicit olInit/olShutDown doesn't happen, so
+// tests have to do it themselves
+
+#include "../common/Fixtures.hpp"
+#include <OffloadAPI.h>
+#include <gtest/gtest.h>
+
+struct olInitTest : ::testing::Test {};
+
+TEST_F(olInitTest, Uninitialized) {
+  ASSERT_ERROR(OL_ERRC_UNINITIALIZED,
+               olIterateDevices(
+                   [](ol_device_handle_t, void *) { return false; }, nullptr));
+}
-- 
cgit v1.2.3


From f242360e156b407902829d694c036b2d22211894 Mon Sep 17 00:00:00 2001
From: Ross Brunton <ross@codeplay.com>
Date: Fri, 20 Jun 2025 15:05:05 +0100
Subject: [Offload] Add type information to device info nodes (#144535)

Rather than being "stringly typed", store values as a std::variant that
can hold various types. This means that liboffload doesn't have to do
any string parsing for integer/bool device info keys.
---
 offload/liboffload/src/OffloadImpl.cpp             | 18 +++++-----
 .../common/include/PluginInterface.h               | 41 +++++++++++++++-------
 2 files changed, 38 insertions(+), 21 deletions(-)

diff --git a/offload/liboffload/src/OffloadImpl.cpp b/offload/liboffload/src/OffloadImpl.cpp
index 8a487563f610..eba8e91ed688 100644
--- a/offload/liboffload/src/OffloadImpl.cpp
+++ b/offload/liboffload/src/OffloadImpl.cpp
@@ -246,23 +246,23 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
   ReturnHelper ReturnValue(PropSize, PropValue, PropSizeRet);
 
   // Find the info if it exists under any of the given names
-  auto GetInfo = [&](std::vector<std::string> Names) {
+  auto GetInfoString = [&](std::vector<std::string> Names) {
     if (Device == OffloadContext::get().HostDevice())
-      return std::string("Host");
+      return "Host";
 
     if (!Device->Device)
-      return std::string("");
+      return "";
 
     auto Info = Device->Device->obtainInfoImpl();
     if (auto Err = Info.takeError())
-      return std::string("");
+      return "";
 
     for (auto Name : Names) {
       if (auto Entry = Info->get(Name))
-        return (*Entry)->Value;
+        return std::get<std::string>((*Entry)->Value).c_str();
     }
 
-    return std::string("");
+    return "";
   };
 
   switch (PropName) {
@@ -273,12 +273,12 @@ Error olGetDeviceInfoImplDetail(ol_device_handle_t Device,
                ? ReturnValue(OL_DEVICE_TYPE_HOST)
                : ReturnValue(OL_DEVICE_TYPE_GPU);
   case OL_DEVICE_INFO_NAME:
-    return ReturnValue(GetInfo({"Device Name"}).c_str());
+    return ReturnValue(GetInfoString({"Device Name"}));
   case OL_DEVICE_INFO_VENDOR:
-    return ReturnValue(GetInfo({"Vendor Name"}).c_str());
+    return ReturnValue(GetInfoString({"Vendor Name"}));
   case OL_DEVICE_INFO_DRIVER_VERSION:
     return ReturnValue(
-        GetInfo({"CUDA Driver Version", "HSA Runtime Version"}).c_str());
+        GetInfoString({"CUDA Driver Version", "HSA Runtime Version"}));
   default:
     return createOffloadError(ErrorCode::INVALID_ENUMERATION,
                               "getDeviceInfo enum '%i' is invalid", PropName);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index f5d995532b7a..91df80030437 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -17,6 +17,7 @@
 #include <list>
 #include <map>
 #include <shared_mutex>
+#include <variant>
 #include <vector>
 
 #include "ExclusiveAccess.h"
@@ -122,7 +123,8 @@ struct InfoTreeNode {
   static constexpr uint64_t IndentSize = 4;
 
   std::string Key;
-  std::string Value;
+  using VariantType = std::variant<uint64_t, std::string, bool, std::monostate>;
+  VariantType Value;
   std::string Units;
   // Need to specify a default value number of elements here as `InfoTreeNode`'s
   // size is unknown. This is a vector (rather than a Key->Value map) since:
@@ -131,15 +133,15 @@ struct InfoTreeNode {
   // * The same key can appear multiple times
   std::unique_ptr<llvm::SmallVector<InfoTreeNode, 8>> Children;
 
-  InfoTreeNode() : InfoTreeNode("", "", "") {}
-  InfoTreeNode(std::string Key, std::string Value, std::string Units)
+  InfoTreeNode() : InfoTreeNode("", std::monostate{}, "") {}
+  InfoTreeNode(std::string Key, VariantType Value, std::string Units)
       : Key(Key), Value(Value), Units(Units) {}
 
   /// Add a new info entry as a child of this node. The entry requires at least
   /// a key string in \p Key. The value in \p Value is optional and can be any
   /// type that is representable as a string. The units in \p Units is optional
   /// and must be a string.
-  template <typename T = std::string>
+  template <typename T = std::monostate>
   InfoTreeNode *add(std::string Key, T Value = T(),
                     const std::string &Units = std::string()) {
     assert(!Key.empty() && "Invalid info key");
@@ -147,15 +149,15 @@ struct InfoTreeNode {
     if (!Children)
       Children = std::make_unique<llvm::SmallVector<InfoTreeNode, 8>>();
 
-    std::string ValueStr;
-    if constexpr (std::is_same_v<T, bool>)
-      ValueStr = Value ? "Yes" : "No";
+    VariantType ValueVariant;
+    if constexpr (std::is_same_v<T, bool> || std::is_same_v<T, std::monostate>)
+      ValueVariant = Value;
     else if constexpr (std::is_arithmetic_v<T>)
-      ValueStr = std::to_string(Value);
+      ValueVariant = static_cast<uint64_t>(Value);
     else
-      ValueStr = Value;
+      ValueVariant = std::string{Value};
 
-    return &Children->emplace_back(Key, ValueStr, Units);
+    return &Children->emplace_back(Key, ValueVariant, Units);
   }
 
   std::optional<InfoTreeNode *> get(StringRef Key) {
@@ -184,8 +186,23 @@ private:
           MaxKeySize - (Key.size() + KeyIndentSize) + IndentSize;
 
       llvm::outs() << std::string(KeyIndentSize, ' ') << Key
-                   << std::string(ValIndentSize, ' ') << Value
-                   << (Units.empty() ? "" : " ") << Units << "\n";
+                   << std::string(ValIndentSize, ' ');
+      std::visit(
+          [](auto &&V) {
+            using T = std::decay_t<decltype(V)>;
+            if constexpr (std::is_same_v<T, std::string>)
+              llvm::outs() << V;
+            else if constexpr (std::is_same_v<T, bool>)
+              llvm::outs() << (V ? "Yes" : "No");
+            else if constexpr (std::is_same_v<T, uint64_t>)
+              llvm::outs() << V;
+            else if constexpr (std::is_same_v<T, std::monostate>) {
+              // Do nothing
+            } else
+              static_assert(false, "doPrint visit not exhaustive");
+          },
+          Value);
+      llvm::outs() << (Units.empty() ? "" : " ") << Units << "\n";
     }
 
     // Print children
-- 
cgit v1.2.3


From 96ab74bf175f46de4b6fbfc68deecd3567e42a52 Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Fri, 20 Jun 2025 10:32:31 -0400
Subject: [InstCombine] remove undef loads, such as memcpy from undef (#143958)

Extend `isAllocSiteRemovable` to be able to check if the ModRef info
indicates the alloca is only Ref or only Mod, and be able to remove it
accordingly. It seemed that there were a surprising number of
benchmarks with this pattern which weren't getting optimized previously
(due to MemorySSA walk limits). There were somewhat more existing tests
than I'd like to have modified which were simply doing exactly this
pattern (and thus relying on undef memory). Claude code contributed the
new tests (and found an important typo that I'd made).

This implements the discussion in
https://github.com/llvm/llvm-project/pull/143782#discussion_r2142720376.
---
 clang/test/Misc/loop-opt-setup.c                   |   2 +-
 .../InstCombine/InstructionCombining.cpp           | 109 ++++++++++++----
 llvm/test/Transforms/InstCombine/and-or-icmps.ll   |  74 +++++++++--
 llvm/test/Transforms/InstCombine/apint-shift.ll    |   3 +-
 .../Transforms/InstCombine/call-cast-target.ll     |  14 +--
 .../test/Transforms/InstCombine/dead-alloc-elim.ll | 140 +++++++++++++++++++++
 llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll |   8 +-
 llvm/test/Transforms/InstCombine/getelementptr.ll  |  17 +--
 llvm/test/Transforms/InstCombine/malloc-free.ll    |   6 +-
 .../multiple-uses-load-bitcast-select.ll           |  20 +--
 llvm/test/Transforms/InstCombine/objsize.ll        |   8 +-
 llvm/test/Transforms/InstCombine/select-load.ll    |  22 +---
 .../InstCombine/shift-amount-reassociation.ll      |   5 +-
 llvm/test/Transforms/InstCombine/vscale_gep.ll     |  12 +-
 14 files changed, 329 insertions(+), 111 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/dead-alloc-elim.ll

diff --git a/clang/test/Misc/loop-opt-setup.c b/clang/test/Misc/loop-opt-setup.c
index 01643e6073b5..c1c620e52200 100644
--- a/clang/test/Misc/loop-opt-setup.c
+++ b/clang/test/Misc/loop-opt-setup.c
@@ -15,7 +15,7 @@ int foo(void) {
 // CHECK-NOT: br i1
 
 void Helper(void) {
-  const int *nodes[5];
+  const int *nodes[5] = {0};
   int num_active = 5;
 
   while (num_active)
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index afd3359e22ff..bcc73090277a 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3277,12 +3277,13 @@ static bool isRemovableWrite(CallBase &CB, Value *UsedV,
   return Dest && Dest->Ptr == UsedV;
 }
 
-static bool isAllocSiteRemovable(Instruction *AI,
-                                 SmallVectorImpl<WeakTrackingVH> &Users,
-                                 const TargetLibraryInfo &TLI) {
+static std::optional<ModRefInfo>
+isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakTrackingVH> &Users,
+                     const TargetLibraryInfo &TLI, bool KnowInit) {
   SmallVector<Instruction*, 4> Worklist;
   const std::optional<StringRef> Family = getAllocationFamily(AI, &TLI);
   Worklist.push_back(AI);
+  ModRefInfo Access = KnowInit ? ModRefInfo::NoModRef : ModRefInfo::Mod;
 
   do {
     Instruction *PI = Worklist.pop_back_val();
@@ -3291,7 +3292,7 @@ static bool isAllocSiteRemovable(Instruction *AI,
       switch (I->getOpcode()) {
       default:
         // Give up the moment we see something we can't handle.
-        return false;
+        return std::nullopt;
 
       case Instruction::AddrSpaceCast:
       case Instruction::BitCast:
@@ -3306,10 +3307,10 @@ static bool isAllocSiteRemovable(Instruction *AI,
         // We also fold comparisons in some conditions provided the alloc has
         // not escaped (see isNeverEqualToUnescapedAlloc).
         if (!ICI->isEquality())
-          return false;
+          return std::nullopt;
         unsigned OtherIndex = (ICI->getOperand(0) == PI) ? 1 : 0;
         if (!isNeverEqualToUnescapedAlloc(ICI->getOperand(OtherIndex), TLI, AI))
-          return false;
+          return std::nullopt;
 
         // Do not fold compares to aligned_alloc calls, as they may have to
         // return null in case the required alignment cannot be satisfied,
@@ -3329,7 +3330,7 @@ static bool isAllocSiteRemovable(Instruction *AI,
         if (CB && TLI.getLibFunc(*CB->getCalledFunction(), TheLibFunc) &&
             TLI.has(TheLibFunc) && TheLibFunc == LibFunc_aligned_alloc &&
             !AlignmentAndSizeKnownValid(CB))
-          return false;
+          return std::nullopt;
         Users.emplace_back(I);
         continue;
       }
@@ -3339,14 +3340,21 @@ static bool isAllocSiteRemovable(Instruction *AI,
         if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
           switch (II->getIntrinsicID()) {
           default:
-            return false;
+            return std::nullopt;
 
           case Intrinsic::memmove:
           case Intrinsic::memcpy:
           case Intrinsic::memset: {
             MemIntrinsic *MI = cast<MemIntrinsic>(II);
-            if (MI->isVolatile() || MI->getRawDest() != PI)
-              return false;
+            if (MI->isVolatile())
+              return std::nullopt;
+            // Note: this could also be ModRef, but we can still interpret that
+            // as just Mod in that case.
+            ModRefInfo NewAccess =
+                MI->getRawDest() == PI ? ModRefInfo::Mod : ModRefInfo::Ref;
+            if ((Access & ~NewAccess) != ModRefInfo::NoModRef)
+              return std::nullopt;
+            Access |= NewAccess;
             [[fallthrough]];
           }
           case Intrinsic::assume:
@@ -3365,11 +3373,6 @@ static bool isAllocSiteRemovable(Instruction *AI,
           }
         }
 
-        if (isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
-          Users.emplace_back(I);
-          continue;
-        }
-
         if (Family && getFreedOperand(cast<CallBase>(I), &TLI) == PI &&
             getAllocationFamily(I, &TLI) == Family) {
           Users.emplace_back(I);
@@ -3383,12 +3386,33 @@ static bool isAllocSiteRemovable(Instruction *AI,
           continue;
         }
 
-        return false;
+        if (!isRefSet(Access) &&
+            isRemovableWrite(*cast<CallBase>(I), PI, TLI)) {
+          Access |= ModRefInfo::Mod;
+          Users.emplace_back(I);
+          continue;
+        }
+
+        return std::nullopt;
 
       case Instruction::Store: {
         StoreInst *SI = cast<StoreInst>(I);
         if (SI->isVolatile() || SI->getPointerOperand() != PI)
-          return false;
+          return std::nullopt;
+        if (isRefSet(Access))
+          return std::nullopt;
+        Access |= ModRefInfo::Mod;
+        Users.emplace_back(I);
+        continue;
+      }
+
+      case Instruction::Load: {
+        LoadInst *LI = cast<LoadInst>(I);
+        if (LI->isVolatile() || LI->getPointerOperand() != PI)
+          return std::nullopt;
+        if (isModSet(Access))
+          return std::nullopt;
+        Access |= ModRefInfo::Ref;
         Users.emplace_back(I);
         continue;
       }
@@ -3396,7 +3420,9 @@ static bool isAllocSiteRemovable(Instruction *AI,
       llvm_unreachable("missing a return?");
     }
   } while (!Worklist.empty());
-  return true;
+
+  assert(Access != ModRefInfo::ModRef);
+  return Access;
 }
 
 Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
@@ -3424,10 +3450,31 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
     DIB.reset(new DIBuilder(*MI.getModule(), /*AllowUnresolved=*/false));
   }
 
-  if (isAllocSiteRemovable(&MI, Users, TLI)) {
+  // Determine what getInitialValueOfAllocation would return without actually
+  // allocating the result.
+  bool KnowInitUndef = false;
+  bool KnowInitZero = false;
+  Constant *Init =
+      getInitialValueOfAllocation(&MI, &TLI, Type::getInt8Ty(MI.getContext()));
+  if (Init) {
+    if (isa<UndefValue>(Init))
+      KnowInitUndef = true;
+    else if (Init->isNullValue())
+      KnowInitZero = true;
+  }
+  // The various sanitizers don't actually return undef memory, but rather
+  // memory initialized with special forms of runtime poison
+  auto &F = *MI.getFunction();
+  if (F.hasFnAttribute(Attribute::SanitizeMemory) ||
+      F.hasFnAttribute(Attribute::SanitizeAddress))
+    KnowInitUndef = false;
+
+  auto Removable =
+      isAllocSiteRemovable(&MI, Users, TLI, KnowInitZero | KnowInitUndef);
+  if (Removable) {
     for (unsigned i = 0, e = Users.size(); i != e; ++i) {
-      // Lowering all @llvm.objectsize calls first because they may
-      // use a bitcast/GEP of the alloca we are removing.
+      // Lowering all @llvm.objectsize and MTI calls first because they may use
+      // a bitcast/GEP of the alloca we are removing.
       if (!Users[i])
        continue;
 
@@ -3444,6 +3491,17 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
           eraseInstFromFunction(*I);
           Users[i] = nullptr; // Skip examining in the next loop.
         }
+        if (auto *MTI = dyn_cast<MemTransferInst>(I)) {
+          if (KnowInitZero && isRefSet(*Removable)) {
+            IRBuilderBase::InsertPointGuard Guard(Builder);
+            Builder.SetInsertPoint(MTI);
+            auto *M = Builder.CreateMemSet(
+                MTI->getRawDest(),
+                ConstantInt::get(Type::getInt8Ty(MI.getContext()), 0),
+                MTI->getLength(), MTI->getDestAlign());
+            M->copyMetadata(*MTI);
+          }
+        }
       }
     }
     for (unsigned i = 0, e = Users.size(); i != e; ++i) {
@@ -3466,7 +3524,14 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
       } else {
         // Casts, GEP, or anything else: we're about to delete this instruction,
         // so it can not have any valid uses.
-        replaceInstUsesWith(*I, PoisonValue::get(I->getType()));
+        Constant *Replace;
+        if (isa<LoadInst>(I)) {
+          assert(KnowInitZero || KnowInitUndef);
+          Replace = KnowInitUndef ? UndefValue::get(I->getType())
+                                  : Constant::getNullValue(I->getType());
+        } else
+          Replace = PoisonValue::get(I->getType());
+        replaceInstUsesWith(*I, Replace);
       }
       eraseInstFromFunction(*I);
     }
diff --git a/llvm/test/Transforms/InstCombine/and-or-icmps.ll b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
index 8824ae48417b..42e502074812 100644
--- a/llvm/test/Transforms/InstCombine/and-or-icmps.ll
+++ b/llvm/test/Transforms/InstCombine/and-or-icmps.ll
@@ -364,23 +364,77 @@ define <2 x i1> @and_ne_with_diff_one_splatvec(<2 x i32> %x) {
 
 define void @simplify_before_foldAndOfICmps(ptr %p) {
 ; CHECK-LABEL: @simplify_before_foldAndOfICmps(
-; CHECK-NEXT:    [[A8:%.*]] = alloca i16, align 2
-; CHECK-NEXT:    [[L7:%.*]] = load i16, ptr [[A8]], align 2
+; CHECK-NEXT:    store i1 true, ptr [[P:%.*]], align 1
+; CHECK-NEXT:    store ptr null, ptr [[P]], align 8
+; CHECK-NEXT:    ret void
+;
+  %A8 = alloca i16
+  %L7 = load i16, ptr %A8
+  %G21 = getelementptr i16, ptr %A8, i8 -1
+  %B11 = udiv i16 %L7, -1
+  %G4 = getelementptr i16, ptr %A8, i16 %B11
+  %L2 = load i16, ptr %G4
+  %L = load i16, ptr %G4
+  %B23 = mul i16 %B11, %B11
+  %L4 = load i16, ptr %A8
+  %B21 = sdiv i16 %L7, %L4
+  %B7 = sub i16 0, %B21
+  %B18 = mul i16 %B23, %B7
+  %C10 = icmp ugt i16 %L, %B11
+  %B20 = and i16 %L7, %L2
+  %B1 = mul i1 %C10, true
+  %C5 = icmp sle i16 %B21, %L
+  %C11 = icmp ule i16 %B21, %L
+  %C7 = icmp slt i16 %B20, 0
+  %B29 = srem i16 %L4, %B18
+  %B15 = add i1 %C7, %C10
+  %B19 = add i1 %C11, %B15
+  %C6 = icmp sge i1 %C11, %B19
+  %B33 = or i16 %B29, %L4
+  %C13 = icmp uge i1 %C5, %B1
+  %C3 = icmp ult i1 %C13, %C6
+  store i16 undef, ptr %G21
+  %C18 = icmp ule i1 %C10, %C7
+  %G26 = getelementptr i1, ptr null, i1 %C3
+  store i16 %B33, ptr %p
+  store i1 %C18, ptr %p
+  store ptr %G26, ptr %p
+  ret void
+}
+
+define void @simplify_before_foldAndOfICmps2(ptr %p, ptr %A8) "instcombine-no-verify-fixpoint" {
+; CHECK-LABEL: @simplify_before_foldAndOfICmps2(
+; CHECK-NEXT:    [[L7:%.*]] = load i16, ptr [[A8:%.*]], align 2
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i16 [[L7]], -1
 ; CHECK-NEXT:    [[B11:%.*]] = zext i1 [[TMP1]] to i16
-; CHECK-NEXT:    [[C10:%.*]] = icmp ugt i16 [[L7]], [[B11]]
-; CHECK-NEXT:    [[C7:%.*]] = icmp slt i16 [[L7]], 0
-; CHECK-NEXT:    [[C3:%.*]] = and i1 [[C7]], [[C10]]
-; CHECK-NEXT:    [[TMP2:%.*]] = xor i1 [[C10]], true
-; CHECK-NEXT:    [[C18:%.*]] = or i1 [[C7]], [[TMP2]]
-; CHECK-NEXT:    [[TMP3:%.*]] = sext i1 [[C3]] to i64
+; CHECK-NEXT:    [[TMP2:%.*]] = zext i1 [[TMP1]] to i64
+; CHECK-NEXT:    [[G4:%.*]] = getelementptr i16, ptr [[A8]], i64 [[TMP2]]
+; CHECK-NEXT:    [[L2:%.*]] = load i16, ptr [[G4]], align 2
+; CHECK-NEXT:    [[L4:%.*]] = load i16, ptr [[A8]], align 2
+; CHECK-NEXT:    [[B21:%.*]] = sdiv i16 [[L7]], [[L4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = select i1 [[TMP1]], i16 [[B21]], i16 0
+; CHECK-NEXT:    [[B18:%.*]] = sub i16 0, [[TMP5]]
+; CHECK-NEXT:    [[C11:%.*]] = icmp ugt i16 [[L2]], [[B11]]
+; CHECK-NEXT:    [[B20:%.*]] = and i16 [[L7]], [[L2]]
+; CHECK-NEXT:    [[C5:%.*]] = icmp sgt i16 [[B21]], [[L2]]
+; CHECK-NEXT:    [[C12:%.*]] = icmp ule i16 [[B21]], [[L2]]
+; CHECK-NEXT:    [[C10:%.*]] = icmp slt i16 [[B20]], 0
+; CHECK-NEXT:    [[B29:%.*]] = srem i16 [[L4]], [[B18]]
+; CHECK-NEXT:    [[B15:%.*]] = xor i1 [[C10]], [[C11]]
+; CHECK-NEXT:    [[TMP6:%.*]] = and i1 [[C12]], [[B15]]
+; CHECK-NEXT:    [[C6:%.*]] = xor i1 [[TMP6]], true
+; CHECK-NEXT:    [[B33:%.*]] = or i16 [[B29]], [[L4]]
+; CHECK-NEXT:    [[C3:%.*]] = and i1 [[C5]], [[C6]]
+; CHECK-NEXT:    [[C4:%.*]] = and i1 [[C3]], [[C11]]
+; CHECK-NEXT:    [[TMP4:%.*]] = xor i1 [[C11]], true
+; CHECK-NEXT:    [[C18:%.*]] = or i1 [[C10]], [[TMP4]]
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i1 [[C4]] to i64
 ; CHECK-NEXT:    [[G26:%.*]] = getelementptr i1, ptr null, i64 [[TMP3]]
-; CHECK-NEXT:    store i16 [[L7]], ptr [[P:%.*]], align 2
+; CHECK-NEXT:    store i16 [[B33]], ptr [[P:%.*]], align 2
 ; CHECK-NEXT:    store i1 [[C18]], ptr [[P]], align 1
 ; CHECK-NEXT:    store ptr [[G26]], ptr [[P]], align 8
 ; CHECK-NEXT:    ret void
 ;
-  %A8 = alloca i16
   %L7 = load i16, ptr %A8
   %G21 = getelementptr i16, ptr %A8, i8 -1
   %B11 = udiv i16 %L7, -1
diff --git a/llvm/test/Transforms/InstCombine/apint-shift.ll b/llvm/test/Transforms/InstCombine/apint-shift.ll
index 3cc530bdbd02..4dd0811bb7ec 100644
--- a/llvm/test/Transforms/InstCombine/apint-shift.ll
+++ b/llvm/test/Transforms/InstCombine/apint-shift.ll
@@ -562,11 +562,10 @@ define i40 @test26(i40 %A) {
 
 ; OSS-Fuzz #9880
 ; https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=9880
-define i177 @ossfuzz_9880(i177 %X) {
+define i177 @ossfuzz_9880(i177 %X, ptr %A) {
 ; CHECK-LABEL: @ossfuzz_9880(
 ; CHECK-NEXT:    ret i177 0
 ;
-  %A = alloca i177
   %L1 = load i177, ptr %A
   %B = or i177 0, -1
   %B5 = udiv i177 %L1, %B
diff --git a/llvm/test/Transforms/InstCombine/call-cast-target.ll b/llvm/test/Transforms/InstCombine/call-cast-target.ll
index 2cedc6c81d73..2f4b4ad2409e 100644
--- a/llvm/test/Transforms/InstCombine/call-cast-target.ll
+++ b/llvm/test/Transforms/InstCombine/call-cast-target.ll
@@ -110,19 +110,17 @@ entry:
 
 declare i1 @fn5(ptr byval({ i32, i32 }) align 4 %r)
 
-define i1 @test5() {
-; CHECK-LABEL: define i1 @test5() {
-; CHECK-NEXT:    [[TMP1:%.*]] = alloca { i32, i32 }, align 4
-; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP1]], align 4
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[TMP1]], i32 4
+define i1 @test5(ptr %ptr) {
+; CHECK-LABEL: define i1 @test5(ptr %ptr) {
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR]], i32 4
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i1 @fn5(i32 [[TMP2]], i32 [[TMP4]])
 ; CHECK-NEXT:    ret i1 [[TMP5]]
 ;
-  %1 = alloca { i32, i32 }, align 4
-  %2 = getelementptr inbounds { i32, i32 }, ptr %1, i32 0, i32 0
+  %2 = getelementptr inbounds { i32, i32 }, ptr %ptr, i32 0, i32 0
   %3 = load i32, ptr %2, align 4
-  %4 = getelementptr inbounds { i32, i32 }, ptr %1, i32 0, i32 1
+  %4 = getelementptr inbounds { i32, i32 }, ptr %ptr, i32 0, i32 1
   %5 = load i32, ptr %4, align 4
   %6 = call i1 @fn5(i32 %3, i32 %5)
   ret i1 %6
diff --git a/llvm/test/Transforms/InstCombine/dead-alloc-elim.ll b/llvm/test/Transforms/InstCombine/dead-alloc-elim.ll
new file mode 100644
index 000000000000..b135f76f709f
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/dead-alloc-elim.ll
@@ -0,0 +1,140 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+declare noalias ptr @calloc(i32, i32) nounwind allockind("alloc,zeroed") allocsize(0,1) "alloc-family"="malloc"
+declare void @free(ptr) allockind("free") "alloc-family"="malloc"
+
+; Test load from uninitialized alloca - should be removed and replaced with undef
+define i32 @test_load_uninitialized_alloca() {
+; CHECK-LABEL: @test_load_uninitialized_alloca(
+; CHECK-NEXT:    ret i32 undef
+;
+  %a = alloca i32
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+; Test load from zero-initialized malloc - should be removed and replaced with zero
+define i32 @test_load_zero_initialized_malloc() {
+; CHECK-LABEL: @test_load_zero_initialized_malloc(
+; CHECK-NEXT:    ret i32 0
+;
+  %a = call ptr @calloc(i32 1, i32 4)
+  %v = load i32, ptr %a
+  call void @free(ptr %a)
+  ret i32 %v
+}
+
+; Test memcpy from uninitialized source - should be removed
+define void @test_memcpy_from_uninitialized_alloca(ptr %dest) {
+; CHECK-LABEL: @test_memcpy_from_uninitialized_alloca(
+; CHECK-NEXT:    ret void
+;
+  %src = alloca i32, align 1
+  call void @llvm.memcpy.p0.p0.i32(ptr %src, ptr %src, i32 4, i1 false)
+  call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 4, i1 false)
+  ret void
+}
+
+; Test memcpy from zeroed source - should transform to memset with zero
+define void @test_memcpy_from_uninitialized_calloc(ptr %dest) {
+; CHECK-LABEL: @test_memcpy_from_uninitialized_calloc(
+; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr noundef nonnull align 1 dereferenceable(16) [[DEST:%.*]], i8 0, i32 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %src = call ptr @calloc(i32 1, i32 16)
+  call void @llvm.memcpy.p0.p0.i32(ptr %dest, ptr %src, i32 16, i1 false)
+  call void @free(ptr %src)
+  ret void
+}
+
+; Test mixed read/write pattern - should not be removable due to write before read
+define i32 @test_write_then_read_alloca() {
+; CHECK-LABEL: @test_write_then_read_alloca(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    store i8 42, ptr [[A]], align 1
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  store i8 42, ptr %a
+  %v = load i32, ptr %a
+  ret i32 %v
+}
+
+; Test read then write pattern - should not be removable due to conflicting access
+define void @test_read_then_write_alloca() {
+; CHECK-LABEL: @test_read_then_write_alloca(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[V:%.*]] = load i32, ptr [[A]], align 4
+; CHECK-NEXT:    [[V8:%.*]] = trunc i32 [[V]] to i8
+; CHECK-NEXT:    store i8 [[V8]], ptr [[A]], align 1
+; CHECK-NEXT:    ret void
+;
+  %a = alloca i32
+  %v = load i32, ptr %a
+  %v8 = trunc i32 %v to i8
+  store i8 %v8, ptr %a
+  ret void
+}
+
+; Test load through GEP from uninitialized alloca
+define i8 @test_load_gep_uninitialized_alloca() {
+; CHECK-LABEL: @test_load_gep_uninitialized_alloca(
+; CHECK-NEXT:    ret i8 undef
+;
+  %a = alloca [4 x i8]
+  %gep = getelementptr [4 x i8], ptr %a, i32 0, i32 2
+  %v = load i8, ptr %gep
+  ret i8 %v
+}
+
+; Test load through bitcast from uninitialized alloca
+define i16 @test_load_bitcast_uninitialized_alloca() {
+; CHECK-LABEL: @test_load_bitcast_uninitialized_alloca(
+; CHECK-NEXT:    ret i16 undef
+;
+  %a = alloca i32
+  %bc = bitcast ptr %a to ptr
+  %v = load i16, ptr %bc
+  ret i16 %v
+}
+
+; Test memmove from zero-initialized malloc
+define void @test_memmove_from_zero_initialized_malloc(ptr %dest) {
+; CHECK-LABEL: @test_memmove_from_zero_initialized_malloc(
+; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr noundef nonnull align 1 dereferenceable(32) [[DEST:%.*]], i8 0, i32 32, i1 false)
+; CHECK-NEXT:    ret void
+;
+  %src = call ptr @calloc(i32 8, i32 4)
+  call void @llvm.memmove.p0.p0.i32(ptr %dest, ptr %src, i32 32, i1 false)
+  call void @free(ptr %src)
+  ret void
+}
+
+; Test multiple loads from same uninitialized alloca
+define { i32, i32 } @test_multiple_loads_uninitialized_alloca() {
+; CHECK-LABEL: @test_multiple_loads_uninitialized_alloca(
+; CHECK-NEXT:    ret { i32, i32 } undef
+;
+  %a = alloca [2 x i32]
+  %gep1 = getelementptr [2 x i32], ptr %a, i32 0, i32 0
+  %gep2 = getelementptr [2 x i32], ptr %a, i32 0, i32 1
+  %v1 = load i32, ptr %gep1
+  %v2 = load i32, ptr %gep2
+  %ret = insertvalue { i32, i32 } { i32 undef, i32 poison }, i32 %v1, 0
+  %ret2 = insertvalue { i32, i32 } %ret, i32 %v2, 1
+  ret { i32, i32 } %ret2
+}
+
+; Test that volatile operations prevent removal
+define i32 @test_volatile_load_prevents_removal() {
+; CHECK-LABEL: @test_volatile_load_prevents_removal(
+; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    [[V:%.*]] = load volatile i32, ptr [[A]], align 4
+; CHECK-NEXT:    ret i32 [[V]]
+;
+  %a = alloca i32
+  %v = load volatile i32, ptr %a
+  ret i32 %v
+}
diff --git a/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll b/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
index 15eb3e15ea44..af18a427ee37 100644
--- a/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
+++ b/llvm/test/Transforms/InstCombine/fp-ret-bitcast.ll
@@ -12,11 +12,11 @@ target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:1
 @"\01L_OBJC_METH_VAR_NAME_112" = internal global [15 x i8] c"whiteComponent\00", section "__TEXT,__cstring,cstring_literals"
 @"\01L_OBJC_SELECTOR_REFERENCES_81" = internal global ptr @"\01L_OBJC_METH_VAR_NAME_112", section "__OBJC,__message_refs,literal_pointers,no_dead_strip"
 
-define void @bork() nounwind  {
+define void @bork(ptr %color, ptr %color.466) nounwind  {
 ; CHECK-LABEL: @bork(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[COLOR:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    [[TMP103:%.*]] = load ptr, ptr [[COLOR]], align 4
+; CHECK-NEXT:    [[TMP103:%.*]] = load ptr, ptr [[COLOR:%.*]], align 4
+; CHECK-NEXT:    store ptr [[TMP103]], ptr [[COLOR_466:%.*]], align 4
 ; CHECK-NEXT:    [[TMP105:%.*]] = load ptr, ptr @"\01L_OBJC_SELECTOR_REFERENCES_81", align 4
 ; CHECK-NEXT:    [[TMP107:%.*]] = call float @objc_msgSend_fpret(ptr [[TMP103]], ptr [[TMP105]]) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    br label [[EXIT:%.*]]
@@ -24,8 +24,6 @@ define void @bork() nounwind  {
 ; CHECK-NEXT:    ret void
 ;
 entry:
-  %color = alloca ptr
-  %color.466 = alloca ptr
   %tmp103 = load ptr, ptr %color, align 4
   store ptr %tmp103, ptr %color.466, align 4
   %tmp105 = load ptr, ptr @"\01L_OBJC_SELECTOR_REFERENCES_81", align 4
diff --git a/llvm/test/Transforms/InstCombine/getelementptr.ll b/llvm/test/Transforms/InstCombine/getelementptr.ll
index e78d70058c14..7568c6edc429 100644
--- a/llvm/test/Transforms/InstCombine/getelementptr.ll
+++ b/llvm/test/Transforms/InstCombine/getelementptr.ll
@@ -580,13 +580,11 @@ define i32 @test20_as1(ptr addrspace(1) %P, i32 %A, i32 %B) {
 }
 
 
-define i32 @test21() {
+define i32 @test21(ptr %pbob1) {
 ; CHECK-LABEL: @test21(
-; CHECK-NEXT:    [[PBOB1:%.*]] = alloca [[INTSTRUCT:%.*]], align 8
-; CHECK-NEXT:    [[RVAL:%.*]] = load i32, ptr [[PBOB1]], align 4
+; CHECK-NEXT:    [[RVAL:%.*]] = load i32, ptr [[PBOB1:%.*]], align 4
 ; CHECK-NEXT:    ret i32 [[RVAL]]
 ;
-  %pbob1 = alloca %intstruct
   %pbob2 = getelementptr %intstruct, ptr %pbob1
   %rval = load i32, ptr %pbob2
   ret i32 %rval
@@ -654,18 +652,16 @@ define i1 @test26(ptr %arr) {
   %struct.siginfo_t = type { i32, i32, i32, { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] } }
   %struct.sigval_t = type { ptr }
 
-define i32 @test27(ptr %to, ptr %from) {
+define i32 @test27(ptr %to, ptr %from, ptr %from_addr) {
 ; CHECK-LABEL: @test27(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[FROM_ADDR:%.*]] = alloca ptr, align 8
-; CHECK-NEXT:    [[T344:%.*]] = load ptr, ptr [[FROM_ADDR]], align 8
+; CHECK-NEXT:    [[T344:%.*]] = load ptr, ptr [[FROM_ADDR:%.*]], align 8
 ; CHECK-NEXT:    [[T348:%.*]] = getelementptr i8, ptr [[T344]], i64 24
 ; CHECK-NEXT:    [[T351:%.*]] = load i32, ptr [[T348]], align 8
 ; CHECK-NEXT:    [[T360:%.*]] = call i32 asm sideeffect "...", "=r,ir,*m,i,0,~{dirflag},~{fpsr},~{flags}"(i32 [[T351]], ptr elementtype([[STRUCT___LARGE_STRUCT:%.*]]) null, i32 -14, i32 0) #[[ATTR0:[0-9]+]]
 ; CHECK-NEXT:    unreachable
 ;
 entry:
-  %from_addr = alloca ptr
   %t344 = load ptr, ptr %from_addr, align 8
   %t345 = getelementptr %struct.siginfo_t, ptr %t344, i32 0, i32 3
   %t346 = getelementptr { { i32, i32, [0 x i8], %struct.sigval_t, i32 }, [88 x i8] }, ptr %t345, i32 0, i32 0
@@ -1345,10 +1341,7 @@ declare noalias ptr @malloc(i64) nounwind allockind("alloc,uninitialized") alloc
 define i32 @test_gep_bitcast_malloc(ptr %a) {
 ; CHECK-LABEL: @test_gep_bitcast_malloc(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[CALL:%.*]] = call noalias dereferenceable_or_null(16) ptr @malloc(i64 16)
-; CHECK-NEXT:    [[G3:%.*]] = getelementptr i8, ptr [[CALL]], i64 12
-; CHECK-NEXT:    [[A_C:%.*]] = load i32, ptr [[G3]], align 4
-; CHECK-NEXT:    ret i32 [[A_C]]
+; CHECK-NEXT:    ret i32 undef
 ;
 entry:
   %call = call noalias ptr @malloc(i64 16) #2
diff --git a/llvm/test/Transforms/InstCombine/malloc-free.ll b/llvm/test/Transforms/InstCombine/malloc-free.ll
index 1f556821270a..989074f97aaf 100644
--- a/llvm/test/Transforms/InstCombine/malloc-free.ll
+++ b/llvm/test/Transforms/InstCombine/malloc-free.ll
@@ -133,17 +133,13 @@ define void @test4() {
 
 define void @test5(ptr %ptr, ptr %esc) {
 ; CHECK-LABEL: @test5(
-; CHECK-NEXT:    [[A:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
-; CHECK-NEXT:    [[B:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
 ; CHECK-NEXT:    [[C:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
 ; CHECK-NEXT:    [[D:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
 ; CHECK-NEXT:    [[E:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
 ; CHECK-NEXT:    [[F:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
 ; CHECK-NEXT:    [[G:%.*]] = call dereferenceable_or_null(700) ptr @malloc(i32 700)
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(32) [[PTR:%.*]], ptr noundef nonnull align 1 dereferenceable(32) [[A]], i32 32, i1 false)
-; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i32(ptr noundef nonnull align 1 dereferenceable(32) [[PTR]], ptr noundef nonnull align 1 dereferenceable(32) [[B]], i32 32, i1 false)
 ; CHECK-NEXT:    store ptr [[C]], ptr [[ESC:%.*]], align 4
-; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[D]], ptr [[PTR]], i32 32, i1 true)
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i32(ptr [[D]], ptr [[PTR:%.*]], i32 32, i1 true)
 ; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i32(ptr [[E]], ptr [[PTR]], i32 32, i1 true)
 ; CHECK-NEXT:    call void @llvm.memset.p0.i32(ptr [[F]], i8 5, i32 32, i1 true)
 ; CHECK-NEXT:    store volatile i8 4, ptr [[G]], align 1
diff --git a/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll b/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
index 38fca0314ae1..5d3512e10f41 100644
--- a/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
+++ b/llvm/test/Transforms/InstCombine/multiple-uses-load-bitcast-select.ll
@@ -1,20 +1,16 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt < %s -passes=instcombine -S -data-layout="E-m:e-i1:8:16-i8:8:16-i64:64-f128:64-v128:64-a:8:16-n32:64" | FileCheck %s
 
-define void @PR35618(ptr %st1, ptr %st2) {
+define void @PR35618(ptr %st1, ptr %st2, ptr %y1, ptr %z1) {
 ; CHECK-LABEL: @PR35618(
-; CHECK-NEXT:    [[Y1:%.*]] = alloca double, align 8
-; CHECK-NEXT:    [[Z1:%.*]] = alloca double, align 8
-; CHECK-NEXT:    [[LD1:%.*]] = load double, ptr [[Y1]], align 8
-; CHECK-NEXT:    [[LD2:%.*]] = load double, ptr [[Z1]], align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load double, ptr [[Y1:%.*]], align 8
+; CHECK-NEXT:    [[LD2:%.*]] = load double, ptr [[Z1:%.*]], align 8
 ; CHECK-NEXT:    [[TMP:%.*]] = fcmp olt double [[LD1]], [[LD2]]
 ; CHECK-NEXT:    [[TMP12_V:%.*]] = select i1 [[TMP]], double [[LD1]], double [[LD2]]
 ; CHECK-NEXT:    store double [[TMP12_V]], ptr [[ST1:%.*]], align 8
 ; CHECK-NEXT:    store double [[TMP12_V]], ptr [[ST2:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
-  %y1 = alloca double
-  %z1 = alloca double
   %ld1 = load double, ptr %y1
   %ld2 = load double, ptr %z1
   %tmp = fcmp olt double %ld1, %ld2
@@ -25,20 +21,16 @@ define void @PR35618(ptr %st1, ptr %st2) {
   ret void
 }
 
-define void @PR35618_asan(ptr %st1, ptr %st2) sanitize_address {
+define void @PR35618_asan(ptr %st1, ptr %st2, ptr %y1, ptr %z1) sanitize_address {
 ; CHECK-LABEL: @PR35618_asan(
-; CHECK-NEXT:    [[Y1:%.*]] = alloca double, align 8
-; CHECK-NEXT:    [[Z1:%.*]] = alloca double, align 8
-; CHECK-NEXT:    [[LD1:%.*]] = load double, ptr [[Y1]], align 8
-; CHECK-NEXT:    [[LD2:%.*]] = load double, ptr [[Z1]], align 8
+; CHECK-NEXT:    [[LD1:%.*]] = load double, ptr [[Y1:%.*]], align 8
+; CHECK-NEXT:    [[LD2:%.*]] = load double, ptr [[Z1:%.*]], align 8
 ; CHECK-NEXT:    [[TMP:%.*]] = fcmp olt double [[LD1]], [[LD2]]
 ; CHECK-NEXT:    [[TMP12_V:%.*]] = select i1 [[TMP]], double [[LD1]], double [[LD2]]
 ; CHECK-NEXT:    store double [[TMP12_V]], ptr [[ST1:%.*]], align 8
 ; CHECK-NEXT:    store double [[TMP12_V]], ptr [[ST2:%.*]], align 8
 ; CHECK-NEXT:    ret void
 ;
-  %y1 = alloca double
-  %z1 = alloca double
   %ld1 = load double, ptr %y1
   %ld2 = load double, ptr %z1
   %tmp = fcmp olt double %ld1, %ld2
diff --git a/llvm/test/Transforms/InstCombine/objsize.ll b/llvm/test/Transforms/InstCombine/objsize.ll
index 1c33412303c1..39f6f493782d 100644
--- a/llvm/test/Transforms/InstCombine/objsize.ll
+++ b/llvm/test/Transforms/InstCombine/objsize.ll
@@ -14,19 +14,17 @@ define i32 @foo() nounwind {
   ret i32 %1
 }
 
-define ptr @bar() nounwind {
+define ptr @bar(ptr %retval) nounwind {
 ; CHECK-LABEL: @bar(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[RETVAL:%.*]] = alloca ptr, align 4
 ; CHECK-NEXT:    br i1 true, label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 ; CHECK:       cond.true:
-; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[RETVAL]], align 4
-; CHECK-NEXT:    ret ptr [[TMP0]]
+; CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[RETVAL:%.*]], align 4
+; CHECK-NEXT:    ret ptr [[TMP1]]
 ; CHECK:       cond.false:
 ; CHECK-NEXT:    ret ptr poison
 ;
 entry:
-  %retval = alloca ptr
   %0 = call i32 @llvm.objectsize.i32.p0(ptr @a, i1 false, i1 false, i1 false)
   %cmp = icmp ne i32 %0, -1
   br i1 %cmp, label %cond.true, label %cond.false
diff --git a/llvm/test/Transforms/InstCombine/select-load.ll b/llvm/test/Transforms/InstCombine/select-load.ll
index 36883423aea3..308dc25bf780 100644
--- a/llvm/test/Transforms/InstCombine/select-load.ll
+++ b/llvm/test/Transforms/InstCombine/select-load.ll
@@ -4,19 +4,14 @@
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-grtev4-linux-gnu"
 
-define i32 @test_plain(i1 %f) {
+define i32 @test_plain(i1 %f, ptr %a, ptr %b) {
 ; CHECK-LABEL: @test_plain(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 8
-; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 8
-; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, ptr [[A]], align 8
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[F:%.*]], ptr [[A:%.*]], ptr [[B1:%.*]]
 ; CHECK-NEXT:    [[B_VAL:%.*]] = load i32, ptr [[B]], align 8
-; CHECK-NEXT:    [[L:%.*]] = select i1 [[F:%.*]], i32 [[A_VAL]], i32 [[B_VAL]]
-; CHECK-NEXT:    ret i32 [[L]]
+; CHECK-NEXT:    ret i32 [[B_VAL]]
 ;
 entry:
-  %a = alloca i32, align 8
-  %b = alloca i32, align 8
   %sel = select i1 %f, ptr %a, ptr %b
   %l = load i32, ptr %sel, align 8
   ret i32 %l
@@ -82,19 +77,14 @@ entry:
 
 ; Msan just propagates shadow, even if speculated load accesses uninitialized
 ; value, instrumentation will select shadow of the desired value anyway.
-define i32 @test_msan(i1 %f) sanitize_memory {
+define i32 @test_msan(i1 %f, ptr %a, ptr %b) sanitize_memory {
 ; CHECK-LABEL: @test_msan(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[A:%.*]] = alloca i32, align 8
-; CHECK-NEXT:    [[B:%.*]] = alloca i32, align 8
-; CHECK-NEXT:    [[A_VAL:%.*]] = load i32, ptr [[A]], align 8
+; CHECK-NEXT:    [[B:%.*]] = select i1 [[F:%.*]], ptr [[A:%.*]], ptr [[B1:%.*]]
 ; CHECK-NEXT:    [[B_VAL:%.*]] = load i32, ptr [[B]], align 8
-; CHECK-NEXT:    [[L:%.*]] = select i1 [[F:%.*]], i32 [[A_VAL]], i32 [[B_VAL]]
-; CHECK-NEXT:    ret i32 [[L]]
+; CHECK-NEXT:    ret i32 [[B_VAL]]
 ;
 entry:
-  %a = alloca i32, align 8
-  %b = alloca i32, align 8
   %sel = select i1 %f, ptr %a, ptr %b
   %l = load i32, ptr %sel, align 8
   ret i32 %l
diff --git a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
index b4c606f037d5..abae2c6b9ab8 100644
--- a/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
+++ b/llvm/test/Transforms/InstCombine/shift-amount-reassociation.ll
@@ -156,11 +156,10 @@ define i32 @t11_shl_nsw_flag_preservation(i32 %x, i32 %y) {
 
 ; Reduced from https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15587
 @X = external global i32
-define i64 @constantexpr() {
-; CHECK-LABEL: @constantexpr(
+define i64 @constantexpr(ptr %A) {
+; CHECK-LABEL: @constantexpr(ptr %A) {
 ; CHECK-NEXT:    ret i64 0
 ;
-  %A = alloca i64
   %L = load i64, ptr %A
   %V = add i64 ptrtoint (ptr @X to i64), 0
   %B2 = shl i64 %V, 0
diff --git a/llvm/test/Transforms/InstCombine/vscale_gep.ll b/llvm/test/Transforms/InstCombine/vscale_gep.ll
index 84019e613d23..5d39ad4e01af 100644
--- a/llvm/test/Transforms/InstCombine/vscale_gep.ll
+++ b/llvm/test/Transforms/InstCombine/vscale_gep.ll
@@ -41,31 +41,27 @@ define void @gep_bitcast(ptr %p) {
 
 ; These tests serve to verify code changes when underlying gep ptr is alloca.
 ; This test is to verify 'inbounds' is added when it's valid to accumulate constant offset.
-define i32 @gep_alloca_inbounds_vscale_zero() {
+define i32 @gep_alloca_inbounds_vscale_zero(ptr %a) {
 ; CHECK-LABEL: @gep_alloca_inbounds_vscale_zero(
-; CHECK-NEXT:    [[A:%.*]] = alloca <vscale x 4 x i32>, align 16
-; CHECK-NEXT:    [[TMP:%.*]] = getelementptr inbounds nuw i8, ptr [[A]], i64 8
+; CHECK-NEXT:    [[TMP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 8
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[TMP]], align 4
 ; CHECK-NEXT:    ret i32 [[LOAD]]
 ;
-  %a = alloca <vscale x 4 x i32>
   %tmp = getelementptr <vscale x 4 x i32>, ptr %a, i32 0, i32 2
   %load = load i32, ptr %tmp
   ret i32 %load
 }
 
 ; This test is to verify 'inbounds' is not added when a constant offset can not be determined at compile-time.
-define i32 @gep_alloca_inbounds_vscale_nonzero() {
+define i32 @gep_alloca_inbounds_vscale_nonzero(ptr %a) {
 ; CHECK-LABEL: @gep_alloca_inbounds_vscale_nonzero(
-; CHECK-NEXT:    [[A:%.*]] = alloca <vscale x 4 x i32>, align 16
 ; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP2:%.*]] = shl nuw i64 [[TMP1]], 4
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP2]]
 ; CHECK-NEXT:    [[TMP:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8
 ; CHECK-NEXT:    [[LOAD:%.*]] = load i32, ptr [[TMP]], align 4
 ; CHECK-NEXT:    ret i32 [[LOAD]]
 ;
-  %a = alloca <vscale x 4 x i32>
   %tmp = getelementptr <vscale x 4 x i32>, ptr %a, i32 1, i32 2
   %load = load i32, ptr %tmp
   ret i32 %load
-- 
cgit v1.2.3


From a2b8a93ff9cfdae4a1578c60fe5efc8ebd8c5571 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 20 Jun 2025 15:46:52 +0100
Subject: [VPlan] Pass NumUnrolledElems as operand to
 VPWidenPointerInductionRecipe. NFC (#119859)

Similarly to VPWidenIntOrFpInductionRecipe, if we want to support it in
EVL tail folding we need to increment the induction by EVL steps instead
of VF*UF steps, but currently this is hard-wired in
VPWidenPointerInductionRecipe.

This adds an operand for the number of elements unrolled and plumbs it
through, so that we can swap it out in
VPlanTransforms::tryAddExplicitVectorLength further down the line.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp |  2 +-
 llvm/lib/Transforms/Vectorize/VPlan.h           | 15 ++++++++++-----
 llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp  | 13 +++++++------
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 88b2ffba1b79..ca1e4cbc6b29 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7826,7 +7826,7 @@ VPHeaderPHIRecipe *VPRecipeBuilder::tryToOptimizeInductionPHI(
     VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, II->getStep(),
                                                            *PSE.getSE());
     return new VPWidenPointerInductionRecipe(
-        Phi, Operands[0], Step, *II,
+        Phi, Operands[0], Step, &Plan.getVFxUF(), *II,
         LoopVectorizationPlanner::getDecisionAndClampRange(
             [&](ElementCount VF) {
               return CM.isScalarAfterVectorization(Phi, VF);
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index ab5ff82a7720..f4163b0743a9 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -2034,25 +2034,30 @@ public:
 };
 
 class VPWidenPointerInductionRecipe : public VPWidenInductionRecipe,
-                                      public VPUnrollPartAccessor<3> {
+                                      public VPUnrollPartAccessor<4> {
   bool IsScalarAfterVectorization;
 
 public:
   /// Create a new VPWidenPointerInductionRecipe for \p Phi with start value \p
-  /// Start.
+  /// Start and the number of elements unrolled \p NumUnrolledElems, typically
+  /// VF*UF.
   VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
+                                VPValue *NumUnrolledElems,
                                 const InductionDescriptor &IndDesc,
                                 bool IsScalarAfterVectorization, DebugLoc DL)
       : VPWidenInductionRecipe(VPDef::VPWidenPointerInductionSC, Phi, Start,
                                Step, IndDesc, DL),
-        IsScalarAfterVectorization(IsScalarAfterVectorization) {}
+        IsScalarAfterVectorization(IsScalarAfterVectorization) {
+    addOperand(NumUnrolledElems);
+  }
 
   ~VPWidenPointerInductionRecipe() override = default;
 
   VPWidenPointerInductionRecipe *clone() override {
     return new VPWidenPointerInductionRecipe(
         cast<PHINode>(getUnderlyingInstr()), getOperand(0), getOperand(1),
-        getInductionDescriptor(), IsScalarAfterVectorization, getDebugLoc());
+        getOperand(2), getInductionDescriptor(), IsScalarAfterVectorization,
+        getDebugLoc());
   }
 
   VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)
@@ -2067,7 +2072,7 @@ public:
   /// the first unrolled part, if it exists. Returns itself if unrolling did not
   /// take place.
   VPValue *getFirstUnrolledPartOperand() {
-    return getUnrollPart(*this) == 0 ? this : getOperand(2);
+    return getUnrollPart(*this) == 0 ? this : getOperand(3);
   }
 
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index f45ce46763c5..39def05b2eac 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -3544,8 +3544,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
   if (CurrentPart == 0) {
     // The recipe represents the first part of the pointer induction. Create the
     // GEP to increment the phi across all unrolled parts.
-    Value *NumUnrolledElems =
-        State.get(&getParent()->getPlan()->getVFxUF(), true);
+    Value *NumUnrolledElems = State.get(getOperand(2), true);
 
     Value *InductionGEP = GetElementPtrInst::Create(
         State.Builder.getInt8Ty(), NewPointerPhi,
@@ -3581,7 +3580,7 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
                                           VPSlotTracker &SlotTracker) const {
-  assert((getNumOperands() == 2 || getNumOperands() == 4) &&
+  assert((getNumOperands() == 3 || getNumOperands() == 5) &&
          "unexpected number of operands");
   O << Indent << "EMIT ";
   printAsOperand(O, SlotTracker);
@@ -3589,11 +3588,13 @@ void VPWidenPointerInductionRecipe::print(raw_ostream &O, const Twine &Indent,
   getStartValue()->printAsOperand(O, SlotTracker);
   O << ", ";
   getStepValue()->printAsOperand(O, SlotTracker);
-  if (getNumOperands() == 4) {
-    O << ", ";
-    getOperand(2)->printAsOperand(O, SlotTracker);
+  O << ", ";
+  getOperand(2)->printAsOperand(O, SlotTracker);
+  if (getNumOperands() == 5) {
     O << ", ";
     getOperand(3)->printAsOperand(O, SlotTracker);
+    O << ", ";
+    getOperand(4)->printAsOperand(O, SlotTracker);
   }
 }
 #endif
-- 
cgit v1.2.3


From 20d57e77f6709ef32791391bc064d3ed5663272a Mon Sep 17 00:00:00 2001
From: Hemang Gadhavi <hemang.gadhavi@ibm.com>
Date: Fri, 20 Jun 2025 20:18:39 +0530
Subject: [lldb][AIX] Added base file for AIX Register Context (#144645)

This PR is in reference to porting LLDB on AIX.
Link to discussions on llvm discourse and github:

1. https://discourse.llvm.org/t/port-lldb-to-ibm-aix/80640
2. https://github.com/llvm/llvm-project/issues/101657
The complete changes for porting are present in this draft PR:
https://github.com/llvm/llvm-project/pull/102601

- Added skeleton for Registercontext file for AIX. (Later we will add
implementation respectively)
---
 lldb/source/Plugins/Process/AIX/CMakeLists.txt     |  1 +
 .../Process/AIX/NativeRegisterContextAIX.cpp       | 54 +++++++++++++++++++
 .../Plugins/Process/AIX/NativeRegisterContextAIX.h | 62 ++++++++++++++++++++++
 3 files changed, 117 insertions(+)
 create mode 100644 lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.cpp
 create mode 100644 lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.h

diff --git a/lldb/source/Plugins/Process/AIX/CMakeLists.txt b/lldb/source/Plugins/Process/AIX/CMakeLists.txt
index 6b3151edbd1e..3a6d9ec118e6 100644
--- a/lldb/source/Plugins/Process/AIX/CMakeLists.txt
+++ b/lldb/source/Plugins/Process/AIX/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_lldb_library(lldbPluginProcessAIX
   NativeProcessAIX.cpp
   NativeThreadAIX.cpp
+  NativeRegisterContextAIX.cpp
 
   LINK_COMPONENTS
     Support
diff --git a/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.cpp b/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.cpp
new file mode 100644
index 000000000000..e44cd7b5a30f
--- /dev/null
+++ b/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.cpp
@@ -0,0 +1,54 @@
+//===---- NativeRegisterContextAIX.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "NativeRegisterContextAIX.h"
+#include "Plugins/Process/AIX/NativeProcessAIX.h"
+
+using namespace lldb_private;
+using namespace lldb_private::process_aix;
+
+lldb::ByteOrder NativeRegisterContextAIX::GetByteOrder() const {
+  return lldb::eByteOrderInvalid;
+}
+
+Status NativeRegisterContextAIX::ReadRegisterRaw(uint32_t reg_index,
+                                                 RegisterValue &reg_value) {
+  return Status("unimplemented");
+}
+
+Status
+NativeRegisterContextAIX::WriteRegisterRaw(uint32_t reg_index,
+                                           const RegisterValue &reg_value) {
+  return Status("unimplemented");
+}
+
+Status NativeRegisterContextAIX::ReadGPR() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::WriteGPR() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::ReadFPR() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::WriteFPR() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::ReadVMX() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::WriteVMX() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::ReadVSX() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::WriteVSX() { return Status("unimplemented"); }
+
+Status NativeRegisterContextAIX::ReadRegisterSet(void *buf, size_t buf_size,
+                                                 unsigned int regset) {
+  return Status("unimplemented");
+}
+
+Status NativeRegisterContextAIX::WriteRegisterSet(void *buf, size_t buf_size,
+                                                  unsigned int regset) {
+  return Status("unimplemented");
+}
diff --git a/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.h b/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.h
new file mode 100644
index 000000000000..e78483a7670f
--- /dev/null
+++ b/lldb/source/Plugins/Process/AIX/NativeRegisterContextAIX.h
@@ -0,0 +1,62 @@
+//===---- NativeRegisterContextAIX.h ----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_PLUGINS_PROCESS_AIX_NATIVEREGISTERCONTEXTAIX_H
+#define LLDB_SOURCE_PLUGINS_PROCESS_AIX_NATIVEREGISTERCONTEXTAIX_H
+
+#include "Plugins/Process/Utility/NativeRegisterContextRegisterInfo.h"
+
+namespace lldb_private::process_aix {
+
+class NativeRegisterContextAIX
+    : public virtual NativeRegisterContextRegisterInfo {
+protected:
+  NativeRegisterContextAIX(NativeThreadProtocol &thread)
+      : NativeRegisterContextRegisterInfo(thread, nullptr) {}
+
+  lldb::ByteOrder GetByteOrder() const;
+
+  virtual Status ReadRegisterRaw(uint32_t reg_index, RegisterValue &reg_value);
+
+  virtual Status WriteRegisterRaw(uint32_t reg_index,
+                                  const RegisterValue &reg_value);
+
+  virtual Status ReadRegisterSet(void *buf, size_t buf_size,
+                                 unsigned int regset);
+
+  virtual Status WriteRegisterSet(void *buf, size_t buf_size,
+                                  unsigned int regset);
+
+  virtual Status ReadGPR();
+
+  virtual Status WriteGPR();
+
+  virtual Status ReadFPR();
+
+  virtual Status WriteFPR();
+
+  virtual Status ReadVMX();
+
+  virtual Status WriteVMX();
+
+  virtual Status ReadVSX();
+
+  virtual Status WriteVSX();
+
+  virtual void *GetGPRBuffer() = 0;
+
+  virtual size_t GetGPRSize() = 0;
+
+  virtual void *GetFPRBuffer() = 0;
+
+  virtual size_t GetFPRSize() = 0;
+};
+
+} // namespace lldb_private::process_aix
+
+#endif // #ifndef LLDB_SOURCE_PLUGINS_PROCESS_AIX_NATIVEREGISTERCONTEXTAIX_H
-- 
cgit v1.2.3


From 95c6c11c747dee61133cff56f1a7ea7445c7ae79 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 15:51:04 +0100
Subject: [X86] combineConcatVectorOps - only always concat logic ops on AVX512
 targets (#145036)

We should only concat logic ops if at least one operand will freely
concatenate. We've now addressed the remaining regressions on AVX2
targets, but still have a number on AVX512 targets which can
aggressively use VPTERNLOG in many cases.
---
 llvm/lib/Target/X86/X86ISelLowering.cpp  |  4 ++--
 llvm/test/CodeGen/X86/vector-fshl-256.ll | 26 +++++++++++++-------------
 llvm/test/CodeGen/X86/vector-fshr-256.ll | 26 +++++++++++++-------------
 3 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index defb7730b4c7..085f44e72476 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -58885,7 +58885,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
     case ISD::OR:
     case ISD::XOR:
     case X86ISD::ANDNP:
-      // TODO: AVX2+ targets should only use CombineSubOperand like AVX1.
+      // TODO: AVX512 targets should only use CombineSubOperand like AVX1/2.
       if (!IsSplat && (VT.is256BitVector() ||
                        (VT.is512BitVector() && Subtarget.useAVX512Regs()))) {
         // Don't concatenate root AVX1 NOT patterns.
@@ -58897,7 +58897,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
           break;
         SDValue Concat0 = CombineSubOperand(VT, Ops, 0);
         SDValue Concat1 = CombineSubOperand(VT, Ops, 1);
-        if (Concat0 || Concat1 || Subtarget.hasInt256())
+        if (Concat0 || Concat1 || Subtarget.useAVX512Regs())
           return DAG.getNode(Opcode, DL, VT,
                              Concat0 ? Concat0 : ConcatSubOperand(VT, Ops, 0),
                              Concat1 ? Concat1 : ConcatSubOperand(VT, Ops, 1));
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index c6e1aa9cd90c..6fbc10307e0b 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -739,15 +739,15 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; XOPAVX2-NEXT:    vpbroadcastb {{.*#+}} xmm6 = [249,249,249,249,249,249,249,249,249,249,249,249,249,249,249,249]
 ; XOPAVX2-NEXT:    vpaddb %xmm6, %xmm5, %xmm7
 ; XOPAVX2-NEXT:    vpshlb %xmm7, %xmm3, %xmm3
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm7, %xmm5
+; XOPAVX2-NEXT:    vpor %xmm3, %xmm5, %xmm3
 ; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm1, %xmm1
 ; XOPAVX2-NEXT:    vpaddb %xmm6, %xmm2, %xmm4
 ; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm3, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm3, %xmm3
 ; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
   ret <32 x i8> %res
@@ -1992,17 +1992,17 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,4,5,6,7,0,7,6,5,4,3,2,1]
 ; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm4
+; XOPAVX2-NEXT:    vpcmpeqd %xmm5, %xmm5, %xmm5
+; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm4, %xmm4
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm6 = [249,250,251,252,253,254,255,0,249,0,255,254,253,252,251,250]
+; XOPAVX2-NEXT:    vpshlb %xmm6, %xmm4, %xmm4
+; XOPAVX2-NEXT:    vpor %xmm4, %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpshlb %xmm6, %xmm1, %xmm1
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; XOPAVX2-NEXT:    vpcmpeqd %xmm3, %xmm3, %xmm3
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm4 = [249,250,251,252,253,254,255,0,249,0,255,254,253,252,251,250]
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <32 x i8> %res
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 9479174d964c..b0a1a91bdccc 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -766,18 +766,18 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
 ; XOPAVX2-NEXT:    vpsubb %xmm4, %xmm5, %xmm6
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm7
 ; XOPAVX2-NEXT:    vpshlb %xmm6, %xmm7, %xmm6
+; XOPAVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; XOPAVX2-NEXT:    vpaddb %xmm7, %xmm7, %xmm7
+; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm7, %xmm4
+; XOPAVX2-NEXT:    vpor %xmm6, %xmm4, %xmm4
 ; XOPAVX2-NEXT:    vpsubb %xmm2, %xmm5, %xmm5
 ; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm6, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vpxor %xmm3, %xmm4, %xmm4
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm5
-; XOPAVX2-NEXT:    vpaddb %xmm5, %xmm5, %xmm5
-; XOPAVX2-NEXT:    vpshlb %xmm4, %xmm5, %xmm4
 ; XOPAVX2-NEXT:    vpxor %xmm3, %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vpshlb %xmm2, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> %amt)
   ret <32 x i8> %res
@@ -1793,16 +1793,16 @@ define <32 x i8> @constant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwind {
 ; XOPAVX2-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,255,254,253,252,251,250,249,0,249,250,251,252,253,254,255]
 ; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm2, %xmm2
+; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; XOPAVX2-NEXT:    vpaddb %xmm4, %xmm4, %xmm4
+; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm5 = [7,6,5,4,3,2,1,0,7,0,1,2,3,4,5,6]
+; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm4, %xmm4
+; XOPAVX2-NEXT:    vpor %xmm2, %xmm4, %xmm2
 ; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm1, %xmm1
-; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm1, %ymm1
-; XOPAVX2-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; XOPAVX2-NEXT:    vpaddb %xmm2, %xmm2, %xmm2
-; XOPAVX2-NEXT:    vmovdqa {{.*#+}} xmm3 = [7,6,5,4,3,2,1,0,7,0,1,2,3,4,5,6]
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm2, %xmm2
 ; XOPAVX2-NEXT:    vpaddb %xmm0, %xmm0, %xmm0
-; XOPAVX2-NEXT:    vpshlb %xmm3, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpshlb %xmm5, %xmm0, %xmm0
+; XOPAVX2-NEXT:    vpor %xmm1, %xmm0, %xmm0
 ; XOPAVX2-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm0
-; XOPAVX2-NEXT:    vpor %ymm1, %ymm0, %ymm0
 ; XOPAVX2-NEXT:    retq
   %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %x, <32 x i8> %y, <32 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1, i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 8, i8 7, i8 6, i8 5, i8 4, i8 3, i8 2, i8 1>)
   ret <32 x i8> %res
-- 
cgit v1.2.3


From b017b4ce9a45d4c5a339e24142da5d4a7e4c5db1 Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Fri, 20 Jun 2025 16:01:55 +0100
Subject: [lldb][DWARF] Remove object_pointer from ParsedDWARFAttributes
 (#144880)

We can just always use `GetCXXObjectParameter` instead. We've only used
this attribute to set the object parameter name on ClangASTMetadata,
which doesn't seem like good enough justification to keep it around.

Depends on https://github.com/llvm/llvm-project/pull/144879
---
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp       | 28 +++++++---------------
 .../Plugins/SymbolFile/DWARF/DWARFASTParserClang.h |  7 +++---
 2 files changed, 12 insertions(+), 23 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 4f79c8aa3f81..3bec89cdf746 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -445,15 +445,6 @@ ParsedDWARFTypeAttributes::ParsedDWARFTypeAttributes(const DWARFDIE &die) {
       name.SetCString(form_value.AsCString());
       break;
 
-    case DW_AT_object_pointer:
-      // GetAttributes follows DW_AT_specification.
-      // DW_TAG_subprogram definitions and declarations may both
-      // have a DW_AT_object_pointer. Don't overwrite the one
-      // we parsed for the definition with the one from the declaration.
-      if (!object_pointer.IsValid())
-        object_pointer = form_value.Reference();
-      break;
-
     case DW_AT_signature:
       signature = form_value;
       break;
@@ -1116,7 +1107,7 @@ bool DWARFASTParserClang::ParseObjCMethod(
 std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
     const DWARFDIE &die, CompilerType clang_type,
     const ParsedDWARFTypeAttributes &attrs, const DWARFDIE &decl_ctx_die,
-    bool is_static, bool &ignore_containing_context) {
+    const DWARFDIE &object_parameter, bool &ignore_containing_context) {
   Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
   SymbolFileDWARF *dwarf = die.GetDWARF();
   assert(dwarf);
@@ -1200,6 +1191,9 @@ std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
       TypeSystemClang::GetDeclContextForType(class_opaque_type), die,
       attrs.name.GetCString());
 
+  // In DWARF, a C++ method is static if it has no object parameter child.
+  const bool is_static = !object_parameter.IsValid();
+
   // We have a C++ member function with no children (this pointer!) and clang
   // will get mad if we try and make a function that isn't well formed in the
   // DWARF, so we will just skip it...
@@ -1225,9 +1219,7 @@ std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
     ClangASTMetadata metadata;
     metadata.SetUserID(die.GetID());
 
-    char const *object_pointer_name =
-        attrs.object_pointer ? attrs.object_pointer.GetName() : nullptr;
-    if (object_pointer_name) {
+    if (char const *object_pointer_name = object_parameter.GetName()) {
       metadata.SetObjectPtrName(object_pointer_name);
       LLDB_LOGF(log, "Setting object pointer name: %s on method object %p.\n",
                 object_pointer_name, static_cast<void *>(cxx_method_decl));
@@ -1323,11 +1315,9 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
         type_handled =
             ParseObjCMethod(*objc_method, die, clang_type, attrs, is_variadic);
       } else if (is_cxx_method) {
-        // In DWARF, a C++ method is static if it has no object parameter child.
-        const bool is_static = !object_parameter.IsValid();
         auto [handled, type_sp] =
-            ParseCXXMethod(die, clang_type, attrs, decl_ctx_die, is_static,
-                           ignore_containing_context);
+            ParseCXXMethod(die, clang_type, attrs, decl_ctx_die,
+                           object_parameter, ignore_containing_context);
         if (type_sp)
           return type_sp;
 
@@ -1422,9 +1412,7 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
           ClangASTMetadata metadata;
           metadata.SetUserID(die.GetID());
 
-          char const *object_pointer_name =
-              attrs.object_pointer ? attrs.object_pointer.GetName() : nullptr;
-          if (object_pointer_name) {
+          if (char const *object_pointer_name = object_parameter.GetName()) {
             metadata.SetObjectPtrName(object_pointer_name);
             LLDB_LOGF(log,
                       "Setting object pointer name: %s on function "
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index 111604ce4068..a90f55bcff94 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -470,7 +470,8 @@ private:
   /// \param[in] decl_ctx_die The DIE representing the DeclContext of the C++
   ///                         method being parsed.
   ///
-  /// \param[in] is_static Is true iff we're parsing a static method.
+  /// \param[in] object_parameter The DIE of this subprogram's object parameter.
+  ///                             May be an invalid DIE for C++ static methods.
   ///
   /// \param[out] ignore_containing_context Will get set to true if the caller
   ///             should treat this C++ method as-if it was not a C++ method.
@@ -485,7 +486,8 @@ private:
                  lldb_private::CompilerType clang_type,
                  const ParsedDWARFTypeAttributes &attrs,
                  const lldb_private::plugin::dwarf::DWARFDIE &decl_ctx_die,
-                 bool is_static, bool &ignore_containing_context);
+                 const lldb_private::plugin::dwarf::DWARFDIE &object_parameter,
+                 bool &ignore_containing_context);
 
   lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
@@ -555,7 +557,6 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
-  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
   lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
   lldb_private::plugin::dwarf::DWARFFormValue containing_type;
   lldb_private::plugin::dwarf::DWARFFormValue signature;
-- 
cgit v1.2.3


From 9dc59cc95b1766510ab43ec62bb087aa9273341a Mon Sep 17 00:00:00 2001
From: Chao Chen <chao.chen@intel.com>
Date: Fri, 20 Jun 2025 10:02:09 -0500
Subject: [MLIR] Incorrect track of usedKey in setPropertiesFromParsedAttr
 (#144789)

co-authored by @chencha3 and @joker-eph
---
 mlir/test/lib/Dialect/Test/TestOps.td  | 2 +-
 mlir/test/mlir-tblgen/op-format.mlir   | 4 ++--
 mlir/tools/mlir-tblgen/OpFormatGen.cpp | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td
index 79bcd9c2e0a9..30234698bc8d 100644
--- a/mlir/test/lib/Dialect/Test/TestOps.td
+++ b/mlir/test/lib/Dialect/Test/TestOps.td
@@ -3179,7 +3179,7 @@ def TestOpWithPropertiesAndInferredType
   ]> {
   let assemblyFormat = "$lhs prop-dict attr-dict";
 
-  let arguments = (ins I32Attr:$lhs, IntProp<"int64_t">:$rhs);
+  let arguments = (ins I32Attr:$lhs, IntProp<"int64_t">:$rhs, OptionalAttr<UnitAttr>: $packed);
   let results = (outs AnyType:$result);
 }
 
diff --git a/mlir/test/mlir-tblgen/op-format.mlir b/mlir/test/mlir-tblgen/op-format.mlir
index 08b0c52413a7..981fb5aff2aa 100644
--- a/mlir/test/mlir-tblgen/op-format.mlir
+++ b/mlir/test/mlir-tblgen/op-format.mlir
@@ -516,8 +516,8 @@ test.format_infer_variadic_type_from_non_variadic %i64, %i64 : i64
 // CHECK: test.with_properties_and_attr 16 < {rhs = 16 : i64}>
 test.with_properties_and_attr 16 <{rhs = 16 : i64}>
 
-// CHECK: test.with_properties_and_inferred_type 16 < {rhs = 16 : i64}>
-%should_be_i32 = test.with_properties_and_inferred_type 16 <{rhs = 16 : i64}>
+// CHECK: test.with_properties_and_inferred_type 16 < {packed, rhs = 16 : i64}>
+%should_be_i32 = test.with_properties_and_inferred_type 16 <{packed, rhs = 16 : i64}>
 // Assert through the verifier that its inferred as i32.
 test.format_all_types_match_var %should_be_i32, %i32 : i32
 
diff --git a/mlir/tools/mlir-tblgen/OpFormatGen.cpp b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
index ef3a18ba7df2..11edf2523f1a 100644
--- a/mlir/tools/mlir-tblgen/OpFormatGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpFormatGen.cpp
@@ -1365,7 +1365,7 @@ if (attr && ::mlir::failed(setFromAttr(prop.{1}, attr, emitError)))
 auto &propStorage = prop.{0};
 auto {0}AttrName = StringAttr::get(ctx, "{0}");
 auto attr = dict.get({0}AttrName);
-usedKeys.insert(StringAttr::get(ctx, "{1}"));
+usedKeys.insert({0}AttrName);
 if (attr || /*isRequired=*/{1}) {{
   if (!attr) {{
     emitError() << "expected key entry for {0} in DictionaryAttr to set "
-- 
cgit v1.2.3


From 05b4bfe19eaba13b4fdf39fc5541077c255b8e3f Mon Sep 17 00:00:00 2001
From: Prajwal Nadig <pnadig@apple.com>
Date: Fri, 20 Jun 2025 16:05:16 +0100
Subject: [ExtractAPI] Include +/- symbols for ObjC methods (#145035)

ObjC methods include a +/- prefix to indicate if they are a class or
instance method. This information is valuable, and must be included in
the navigator generated by ExtractAPI.

rdar://150870936
---
 .../ExtractAPI/Serialization/SymbolGraphSerializer.cpp   | 16 ++++++++++++++++
 clang/test/ExtractAPI/objc_instancetype.m                |  8 ++++++++
 2 files changed, 24 insertions(+)

diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index 139023f32e8d..d3df9eb604f2 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -341,6 +341,22 @@ Object serializeNames(const APIRecord *Record) {
   serializeArray(Names, "subHeading",
                  serializeDeclarationFragments(Record->SubHeading));
   DeclarationFragments NavigatorFragments;
+  // The +/- prefix for Objective-C methods is important information, and
+  // should be included in the navigator fragment. The entire subheading is
+  // not included as it can contain too much information for other records.
+  switch (Record->getKind()) {
+  case APIRecord::RK_ObjCClassMethod:
+    NavigatorFragments.append("+ ", DeclarationFragments::FragmentKind::Text,
+                              /*PreciseIdentifier*/ "");
+    break;
+  case APIRecord::RK_ObjCInstanceMethod:
+    NavigatorFragments.append("- ", DeclarationFragments::FragmentKind::Text,
+                              /*PreciseIdentifier*/ "");
+    break;
+  default:
+    break;
+  }
+
   NavigatorFragments.append(Record->Name,
                             DeclarationFragments::FragmentKind::Identifier,
                             /*PreciseIdentifier*/ "");
diff --git a/clang/test/ExtractAPI/objc_instancetype.m b/clang/test/ExtractAPI/objc_instancetype.m
index 071ebe440918..dbd47a1f746f 100644
--- a/clang/test/ExtractAPI/objc_instancetype.m
+++ b/clang/test/ExtractAPI/objc_instancetype.m
@@ -157,6 +157,10 @@
       },
       "names": {
         "navigator": [
+          {
+            "kind": "text",
+            "spelling": "- "
+          },
           {
             "kind": "identifier",
             "spelling": "init"
@@ -228,6 +232,10 @@
       },
       "names": {
         "navigator": [
+          {
+            "kind": "text",
+            "spelling": "- "
+          },
           {
             "kind": "identifier",
             "spelling": "reset"
-- 
cgit v1.2.3


From 71e20c6c86e04863df80e286a004a20070a5a610 Mon Sep 17 00:00:00 2001
From: Douglas <Douglas.Gliner@sony.com>
Date: Fri, 20 Jun 2025 08:13:48 -0700
Subject: Fix references to required libraries when building LLVM with ASAN and
 MultiThreaded[Debug] on Windows (#139657)

After https://github.com/llvm/llvm-project/pull/81677, statically
linking ASAN under Windows is no longer supported. Therefore, when using
Clang built past
https://github.com/llvm/llvm-project/commit/53a81d4d26f0409de8a0655d7af90f2bea222a12
to build LLVM / Clang with
`-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded[Debug]
-DLLVM_USE_SANITIZER=Address`, a different set of dependent libraries
must be linked. This is mentioned in the description of
https://github.com/llvm/llvm-project/pull/81677 and also in
https://devblogs.microsoft.com/cppblog/msvc-address-sanitizer-one-dll-for-all-runtime-configurations/.
---
 llvm/cmake/modules/HandleLLVMOptions.cmake | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake
index 8004d3571fc8..5aa047a33ba6 100644
--- a/llvm/cmake/modules/HandleLLVMOptions.cmake
+++ b/llvm/cmake/modules/HandleLLVMOptions.cmake
@@ -1038,13 +1038,14 @@ if(LLVM_USE_SANITIZER)
         # lld string tail merging interacts badly with ASAN on Windows, turn it off here
         # See https://github.com/llvm/llvm-project/issues/62078
         append("/opt:nolldtailmerge" CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
+        # Static and dynamic C runtimes all load ASAN as a DLL
+        # See https://devblogs.microsoft.com/cppblog/msvc-address-sanitizer-one-dll-for-all-runtime-configurations/
+        append("clang_rt.asan_dynamic-${arch}.lib" CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
         if (${CMAKE_MSVC_RUNTIME_LIBRARY} MATCHES "^(MultiThreaded|MultiThreadedDebug)$")
-          append("/wholearchive:clang_rt.asan-${arch}.lib /wholearchive:clang_rt.asan_cxx-${arch}.lib"
-            CMAKE_EXE_LINKER_FLAGS)
-          append("/wholearchive:clang_rt.asan_dll_thunk-${arch}.lib"
-            CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
+          append("/wholearchive:clang_rt.asan_static_runtime_thunk-${arch}.lib"
+            CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
         else()
-          append("clang_rt.asan_dynamic-${arch}.lib /wholearchive:clang_rt.asan_dynamic_runtime_thunk-${arch}.lib"
+          append("/wholearchive:clang_rt.asan_dynamic_runtime_thunk-${arch}.lib"
             CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS)
         endif()
       endif()
-- 
cgit v1.2.3


From b533b0ec34ac36d8a6af406d1fb046e07f95f717 Mon Sep 17 00:00:00 2001
From: Jeremy Kun <jkun@google.com>
Date: Fri, 20 Jun 2025 08:16:52 -0700
Subject: Define a DataFlowSolver helper that loads sensible default analyses
 (#143415)

Cf. https://discourse.llvm.org/t/mlir-dead-code-analysis/67568/10

Custom analysis passes will not work properly unless both
DeadCodeAnalysis and SparseConstantPropagation are loaded to the
DataFlowSolver. This is intended behavior, but surprising to many users
as shown in the thread. In lieu of a longer-term fix (which I am not
knowledgeable enough to implement myself, yet), this commit adds a
helper function that loads these two analyses, as well as providing
breadcrumbs for an explanation of the problem. The existing places in
the codebase where these two analyses are loaded for the purpose of
running other unrelated analyses are replaced by the use of the helper.

---------

Co-authored-by: Jeremy Kun <j2kun@users.noreply.github.com>
Co-authored-by: Oleksandr "Alex" Zinenko <azinenko@amd.com>
---
 mlir/include/mlir/Analysis/DataFlow/Utils.h        | 37 ++++++++++++++++++++++
 mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp    |  6 ++--
 .../XeGPU/Transforms/XeGPUSubgroupDistribute.cpp   |  6 ++--
 .../DataFlow/TestDenseBackwardDataFlowAnalysis.cpp |  6 ++--
 .../DataFlow/TestDenseForwardDataFlowAnalysis.cpp  |  6 ++--
 .../TestSparseBackwardDataFlowAnalysis.cpp         |  6 ++--
 6 files changed, 47 insertions(+), 20 deletions(-)
 create mode 100644 mlir/include/mlir/Analysis/DataFlow/Utils.h

diff --git a/mlir/include/mlir/Analysis/DataFlow/Utils.h b/mlir/include/mlir/Analysis/DataFlow/Utils.h
new file mode 100644
index 000000000000..e97f2f70f609
--- /dev/null
+++ b/mlir/include/mlir/Analysis/DataFlow/Utils.h
@@ -0,0 +1,37 @@
+//===-Utils.h - DataFlow utility functions ----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines utility functions for dataflow analyses.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_ANALYSIS_DATAFLOW_UTILS_H
+#define MLIR_ANALYSIS_DATAFLOW_UTILS_H
+
+#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
+#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
+#include "mlir/Analysis/DataFlowFramework.h"
+
+namespace mlir {
+namespace dataflow {
+
+/// Populates a DataFlowSolver with analyses that are required to ensure
+/// user-defined analyses are run properly.
+///
+/// This helper is intended to be an interim fix until a more robust solution
+/// can be implemented in the DataFlow framework directly. Cf.
+/// https://discourse.llvm.org/t/mlir-dead-code-analysis/67568
+inline void loadBaselineAnalyses(DataFlowSolver &solver) {
+  solver.load<dataflow::DeadCodeAnalysis>();
+  solver.load<dataflow::SparseConstantPropagation>();
+}
+
+} // end namespace dataflow
+} // end namespace mlir
+
+#endif // MLIR_ANALYSIS_DATAFLOW_INTEGERANGEANALYSIS_H
diff --git a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
index d61cdb143e7d..24a78400eb84 100644
--- a/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
+++ b/mlir/lib/Analysis/DataFlow/LivenessAnalysis.cpp
@@ -10,9 +10,8 @@
 #include <cassert>
 #include <mlir/Analysis/DataFlow/LivenessAnalysis.h>
 
-#include <mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h>
-#include <mlir/Analysis/DataFlow/DeadCodeAnalysis.h>
 #include <mlir/Analysis/DataFlow/SparseAnalysis.h>
+#include <mlir/Analysis/DataFlow/Utils.h>
 #include <mlir/Analysis/DataFlowFramework.h>
 #include <mlir/IR/Operation.h>
 #include <mlir/IR/Value.h>
@@ -249,8 +248,7 @@ void LivenessAnalysis::setToExitState(Liveness *lattice) {
 RunLivenessAnalysis::RunLivenessAnalysis(Operation *op) {
   SymbolTableCollection symbolTable;
 
-  solver.load<DeadCodeAnalysis>();
-  solver.load<SparseConstantPropagation>();
+  loadBaselineAnalyses(solver);
   solver.load<LivenessAnalysis>(symbolTable);
   (void)solver.initializeAndRun(op);
 }
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index c84906cc4556..66d21dbdaf06 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -5,9 +5,8 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
-#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/SparseAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
 #include "mlir/Analysis/DataFlowFramework.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Utils/DistributionUtils.h"
@@ -620,8 +619,7 @@ public:
 
   RunLayoutInfoPropagation(Operation *op) : target(op) {
     SymbolTableCollection symbolTable;
-    solver.load<DeadCodeAnalysis>();
-    solver.load<SparseConstantPropagation>();
+    loadBaselineAnalyses(solver);
     solver.load<LayoutInfoPropagation>(symbolTable);
     (void)solver.initializeAndRun(op);
   }
diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
index da543f4f04f9..d57b41c41de6 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestDenseBackwardDataFlowAnalysis.cpp
@@ -13,9 +13,8 @@
 #include "TestDenseDataFlowAnalysis.h"
 #include "TestDialect.h"
 #include "TestOps.h"
-#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
-#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/DenseAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
 #include "mlir/Analysis/DataFlowFramework.h"
 #include "mlir/IR/Builders.h"
 #include "mlir/IR/SymbolTable.h"
@@ -283,9 +282,8 @@ struct TestNextAccessPass
 
     auto config = DataFlowConfig().setInterprocedural(interprocedural);
     DataFlowSolver solver(config);
-    solver.load<DeadCodeAnalysis>();
+    loadBaselineAnalyses(solver);
     solver.load<NextAccessAnalysis>(symbolTable, assumeFuncReads);
-    solver.load<SparseConstantPropagation>();
     solver.load<UnderlyingValueAnalysis>();
     if (failed(solver.initializeAndRun(op))) {
       emitError(op->getLoc(), "dataflow solver failed");
diff --git a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
index f4f8e9115a3f..a88ed7f8dea8 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestDenseForwardDataFlowAnalysis.cpp
@@ -13,9 +13,8 @@
 #include "TestDenseDataFlowAnalysis.h"
 #include "TestDialect.h"
 #include "TestOps.h"
-#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
-#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/DenseAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Pass/Pass.h"
 #include "mlir/Support/LLVM.h"
@@ -240,8 +239,7 @@ struct TestLastModifiedPass
     Operation *op = getOperation();
 
     DataFlowSolver solver(DataFlowConfig().setInterprocedural(interprocedural));
-    solver.load<DeadCodeAnalysis>();
-    solver.load<SparseConstantPropagation>();
+    loadBaselineAnalyses(solver);
     solver.load<LastModifiedAnalysis>(assumeFuncWrites);
     solver.load<UnderlyingValueAnalysis>();
     if (failed(solver.initializeAndRun(op)))
diff --git a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
index 4b02865b6ae1..0bdb7c25c3b5 100644
--- a/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
+++ b/mlir/test/lib/Analysis/DataFlow/TestSparseBackwardDataFlowAnalysis.cpp
@@ -6,9 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
-#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
 #include "mlir/Analysis/DataFlow/SparseAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/Interfaces/SideEffectInterfaces.h"
 #include "mlir/Pass/Pass.h"
@@ -182,8 +181,7 @@ struct TestWrittenToPass
     SymbolTableCollection symbolTable;
 
     DataFlowSolver solver(DataFlowConfig().setInterprocedural(interprocedural));
-    solver.load<DeadCodeAnalysis>();
-    solver.load<SparseConstantPropagation>();
+    loadBaselineAnalyses(solver);
     solver.load<WrittenToAnalysis>(symbolTable, assumeFuncWrites);
     if (failed(solver.initializeAndRun(op)))
       return signalPassFailure();
-- 
cgit v1.2.3


From c734377544fc5a854c539fafc9b9b658f12230a3 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 16:20:03 +0100
Subject: [DAG] foldMaskedMerge - fix Wparentheses operator precedence warning.
 NFC.

---
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index a6b9cc81edde..701a76c4cc6b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -7214,9 +7214,8 @@ static SDValue foldMaskedMerge(SDNode *Node, SelectionDAG &DAG,
                                const TargetLowering &TLI, const SDLoc &DL) {
   // Note that masked-merge variants using XOR or ADD expressions are
   // normalized to OR by InstCombine so we only check for OR or AND.
-  assert(Node->getOpcode() == ISD::OR ||
-         Node->getOpcode() == ISD::AND &&
-             "Must be called with ISD::OR or ISD::AND node");
+  assert((Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::AND) &&
+         "Must be called with ISD::OR or ISD::AND node");
 
   // If the target supports and-not, don't fold this.
   if (TLI.hasAndNot(SDValue(Node, 0)))
-- 
cgit v1.2.3


From 3f1de197b1c339b311329c02bb739860b32c073f Mon Sep 17 00:00:00 2001
From: Joseph Huber <huberjn@outlook.com>
Date: Fri, 20 Jun 2025 10:31:54 -0500
Subject: [Offload] Rework compiling device code for unit test suites (#144776)

Summary:
I'll probably want to use this as a more generic utility in the future.
This patch reworks it to make it a top level function. I also tried to
decouple this from the OpenMP utilities to make that easier in the
future. Instead, I just use `-march=native` functionality which is the
same thing. Needed a small hack to skip the linker stage for checking if
that works.

This should still create the same output as far as I'm aware.
---
 offload/unittests/CMakeLists.txt                   | 73 +++++++++++++++++++++-
 .../OffloadAPI/device_code/CMakeLists.txt          | 69 +-------------------
 2 files changed, 74 insertions(+), 68 deletions(-)

diff --git a/offload/unittests/CMakeLists.txt b/offload/unittests/CMakeLists.txt
index 985dd892d804..7cd41e1dcdaf 100644
--- a/offload/unittests/CMakeLists.txt
+++ b/offload/unittests/CMakeLists.txt
@@ -1,6 +1,72 @@
 add_custom_target(OffloadUnitTests)
 set_target_properties(OffloadUnitTests PROPERTIES FOLDER "Tests/UnitTests")
 
+function(add_offload_test_device_code test_filename test_name)
+  set(SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${test_filename})
+  set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)
+
+  # Try to build with support for NVPTX devices.
+  if("cuda" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
+    find_package(CUDAToolkit QUIET)
+    if(CUDAToolkit_FOUND)
+      get_filename_component(cuda_path "${CUDAToolkit_BIN_DIR}" DIRECTORY ABSOLUTE)
+    endif()
+    check_cxx_compiler_flag(
+      "--target=nvptx64-nvidia-cuda -march=native --cuda-path=${cuda_path}" PLATFORM_HAS_NVPTX)
+
+    if(OFFLOAD_TESTS_FORCE_NVPTX_ARCH)
+      set(nvptx_arch "${OFFLOAD_TESTS_FORCE_NVPTX_ARCH}")
+    elseif(PLATFORM_HAS_NVPTX)
+      set(nvptx_arch "native")
+    endif()
+
+    if(nvptx_arch AND CUDAToolkit_FOUND)
+      set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${test_name}.nvptx64.bin")
+      add_custom_command(
+        OUTPUT ${output_file}
+        COMMAND ${CMAKE_C_COMPILER}
+        --target=nvptx64-nvidia-cuda -march=${nvptx_arch}
+        -nogpulib --cuda-path=${CUDA_ROOT} -flto ${ARGN}
+        -c ${SRC_PATH} -o ${output_file}
+        DEPENDS ${SRC_PATH}
+      )
+      add_custom_target(${test_name}.nvptx64 DEPENDS ${output_file})
+    endif()
+  endif()
+
+  # Try to build with support for AMDGPU devices.
+  if("amdgpu" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
+    check_cxx_compiler_flag("--target=amdgcn-amd-amdhsa -mcpu=native" PLATFORM_HAS_AMDGPU)
+
+    if(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH)
+      set(amdgpu_arch "${OFFLOAD_TESTS_FORCE_AMDGPU_ARCH}")
+    elseif(PLATFORM_HAS_AMDGPU)
+      set(amdgpu_arch "native")
+    endif()
+
+    if(amdgpu_arch)
+      set(output_file "${CMAKE_CURRENT_BINARY_DIR}/${test_name}.amdgpu.bin")
+      add_custom_command(
+        OUTPUT ${output_file}
+        COMMAND ${CMAKE_C_COMPILER}
+        --target=amdgcn-amd-amdhsa -mcpu=${amdgpu_arch}
+        -nogpulib -flto ${ARGN} -c ${SRC_PATH} -o ${output_file}
+        DEPENDS ${SRC_PATH}
+      )
+      add_custom_target(${test_name}.amdgpu DEPENDS ${output_file})
+    endif()
+  endif()
+
+  # Create a single dependency target for the device code.
+  add_custom_target(${test_name}.bin)
+  if(TARGET ${test_name}.amdgpu)
+    add_dependencies(${test_name}.bin ${test_name}.amdgpu)
+  endif()
+  if(TARGET ${test_name}.nvptx64)
+    add_dependencies(${test_name}.bin ${test_name}.nvptx64)
+  endif()
+endfunction()
+
 function(add_offload_unittest test_dirname)
   set(target_name "${test_dirname}.unittests")
 
@@ -9,10 +75,15 @@ function(add_offload_unittest test_dirname)
   add_unittest(OffloadUnitTests "${target_name}"
     ${CMAKE_CURRENT_SOURCE_DIR}/common/Environment.cpp
     ${files})
-  add_dependencies(${target_name} ${PLUGINS_TEST_COMMON} OffloadUnitTestsDeviceBins)
+  add_dependencies(${target_name} ${PLUGINS_TEST_COMMON} offload_device_binaries)
   target_compile_definitions(${target_name} PRIVATE DEVICE_CODE_PATH="${OFFLOAD_TEST_DEVICE_CODE_PATH}")
   target_link_libraries(${target_name} PRIVATE ${PLUGINS_TEST_COMMON})
   target_include_directories(${target_name} PRIVATE ${PLUGINS_TEST_INCLUDE})
 endfunction()
 
+set(OFFLOAD_TESTS_FORCE_NVPTX_ARCH "" CACHE STRING
+  "Force building of NVPTX device code for Offload unit tests with the given arch, e.g. sm_61")
+set(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH "" CACHE STRING
+  "Force building of AMDGPU device code for Offload unit tests with the given arch, e.g. gfx1030")
+
 add_subdirectory(OffloadAPI)
diff --git a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
index c2e4d0cb24e6..132c7a7c51fb 100644
--- a/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
+++ b/offload/unittests/OffloadAPI/device_code/CMakeLists.txt
@@ -1,72 +1,7 @@
-macro(add_offload_test_device_code test_filename test_name)
-    set(SRC_PATH ${CMAKE_CURRENT_SOURCE_DIR}/${test_filename})
-
-    # Build for NVPTX
-    if(OFFLOAD_TEST_TARGET_NVIDIA)
-        set(BIN_PATH ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.nvptx64.bin)
-        add_custom_command(OUTPUT ${BIN_PATH}
-            COMMAND
-            ${CMAKE_C_COMPILER} --target=nvptx64-nvidia-cuda
-            ${ARGN}
-            -march=${LIBOMPTARGET_DEP_CUDA_ARCH}
-            --cuda-path=${CUDA_ROOT}
-            ${SRC_PATH} -o ${BIN_PATH}
-            DEPENDS ${SRC_PATH}
-        )
-        list(APPEND BIN_PATHS ${BIN_PATH})
-    endif()
-
-    # Build for AMDGPU
-    if(OFFLOAD_TEST_TARGET_AMDGPU)
-        set(BIN_PATH ${CMAKE_CURRENT_BINARY_DIR}/${test_name}.amdgpu.bin)
-        add_custom_command(OUTPUT ${BIN_PATH}
-            COMMAND
-            ${CMAKE_C_COMPILER} --target=amdgcn-amd-amdhsa -nogpulib
-            ${ARGN}
-            -mcpu=${LIBOMPTARGET_DEP_AMDGPU_ARCH}
-            ${SRC_PATH} -o ${BIN_PATH}
-            DEPENDS ${SRC_PATH}
-        )
-        list(APPEND BIN_PATHS ${BIN_PATH})
-    endif()
-
-    # TODO: Build for host CPU
-endmacro()
-
-
-# Decide what device targets to build for. LibomptargetGetDependencies is
-# included at the top-level so the GPUs present on the system are already
-# detected.
-set(OFFLOAD_TESTS_FORCE_NVIDIA_ARCH "" CACHE STRING
-    "Force building of NVPTX device code for Offload unit tests with the given arch, e.g. sm_61")
-set(OFFLOAD_TESTS_FORCE_AMDGPU_ARCH "" CACHE STRING
-    "Force building of AMDGPU device code for Offload unit tests with the given arch, e.g. gfx1030")
-
-find_package(CUDAToolkit QUIET)
-if(CUDAToolkit_FOUND)
-  get_filename_component(CUDA_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY ABSOLUTE)
-endif()
-if (OFFLOAD_TESTS_FORCE_NVIDIA_ARCH)
-    set(LIBOMPTARGET_DEP_CUDA_ARCH ${OFFLOAD_TESTS_FORCE_NVIDIA_ARCH})
-    set(OFFLOAD_TEST_TARGET_NVIDIA ON)
-elseif (LIBOMPTARGET_FOUND_NVIDIA_GPU AND CUDA_ROOT AND "cuda" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
-    set(OFFLOAD_TEST_TARGET_NVIDIA ON)
-endif()
-
-if (OFFLOAD_TESTS_FORCE_AMDGPU_ARCH)
-    set(LIBOMPTARGET_DEP_AMDGPU_ARCH ${OFFLOAD_TESTS_FORCE_AMDGPU_ARCH})
-    set(OFFLOAD_TEST_TARGET_AMDGPU ON)
-elseif (LIBOMPTARGET_FOUND_AMDGPU_GPU AND "amdgpu" IN_LIST LIBOMPTARGET_PLUGINS_TO_BUILD)
-    list(GET LIBOMPTARGET_AMDGPU_DETECTED_ARCH_LIST 0 LIBOMPTARGET_DEP_AMDGPU_ARCH)
-    set(OFFLOAD_TEST_TARGET_AMDGPU ON)
-endif()
-
 add_offload_test_device_code(foo.c foo)
 add_offload_test_device_code(bar.c bar)
-# By default, amdhsa will add a number of "hidden" arguments to the kernel defintion
-# O3 disables this, and results in a kernel function with actually no arguments as seen by liboffload
+# Compile with optimizations to eliminate AMDGPU implicit arguments.
 add_offload_test_device_code(noargs.c noargs -O3)
 
-add_custom_target(OffloadUnitTestsDeviceBins DEPENDS ${BIN_PATHS})
-
+add_custom_target(offload_device_binaries DEPENDS foo.bin bar.bin noargs.bin)
 set(OFFLOAD_TEST_DEVICE_CODE_PATH ${CMAKE_CURRENT_BINARY_DIR} PARENT_SCOPE)
-- 
cgit v1.2.3


From 887222e3526fbe08e748a33f740296ac22bf1ab1 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Fri, 20 Jun 2025 08:33:59 -0700
Subject: [mlir] Migrate away from ArrayRef(std::nullopt) (NFC) (#144989)

ArrayRef has a constructor that accepts std::nullopt.  This
constructor dates back to the days when we still had llvm::Optional.

Since the use of std::nullopt outside the context of std::optional is
kind of abuse and not intuitive to new comers, I would like to move
away from the constructor and eventually remove it.

This patch takes care of the mlir side of the migration, starting with
straightforward places where I see ArrayRef or ValueRange nearby.
Note that ValueRange has a constructor that forwards arguments to an
ArrayRef constructor.
---
 mlir/include/mlir/AsmParser/AsmParserState.h         | 2 +-
 mlir/include/mlir/Dialect/Linalg/Utils/Utils.h       | 2 +-
 mlir/include/mlir/Dialect/Tensor/Utils/Utils.h       | 2 +-
 mlir/include/mlir/ExecutionEngine/ExecutionEngine.h  | 3 +--
 mlir/include/mlir/IR/BlockSupport.h                  | 2 +-
 mlir/include/mlir/IR/Builders.h                      | 7 +++----
 mlir/include/mlir/IR/BuiltinTypes.td                 | 4 ++--
 mlir/include/mlir/IR/PatternMatch.h                  | 7 +++----
 mlir/include/mlir/IR/Region.h                        | 2 +-
 mlir/include/mlir/IR/SymbolTable.h                   | 3 ++-
 mlir/include/mlir/IR/TypeRange.h                     | 2 +-
 mlir/include/mlir/IR/ValueRange.h                    | 4 ++--
 mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h  | 7 +++----
 mlir/include/mlir/Tools/PDLL/AST/Types.h             | 3 +--
 mlir/include/mlir/Transforms/DialectConversion.h     | 2 +-
 mlir/include/mlir/Transforms/Passes.h                | 4 ++--
 mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp | 2 +-
 mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h              | 2 +-
 mlir/lib/Pass/PassStatistics.cpp                     | 2 +-
 mlir/unittests/IR/OperationSupportTest.cpp           | 5 ++---
 mlir/unittests/IR/ValueTest.cpp                      | 5 ++---
 21 files changed, 33 insertions(+), 39 deletions(-)

diff --git a/mlir/include/mlir/AsmParser/AsmParserState.h b/mlir/include/mlir/AsmParser/AsmParserState.h
index 98bdc4696b84..e47cb32ee402 100644
--- a/mlir/include/mlir/AsmParser/AsmParserState.h
+++ b/mlir/include/mlir/AsmParser/AsmParserState.h
@@ -195,7 +195,7 @@ public:
   /// Finalize the most recently started operation definition.
   void finalizeOperationDefinition(
       Operation *op, SMRange nameLoc, SMLoc endLoc,
-      ArrayRef<std::pair<unsigned, SMLoc>> resultGroups = std::nullopt);
+      ArrayRef<std::pair<unsigned, SMLoc>> resultGroups = {});
 
   /// Start a definition for a region nested under the current operation.
   void startRegionDefinition();
diff --git a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
index fc151d02ceef..48978eb7663d 100644
--- a/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Linalg/Utils/Utils.h
@@ -94,7 +94,7 @@ bool isReductionIterator(utils::IteratorType iteratorType);
 /// ```
 Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type,
                             Value source, Value padding, bool nofold,
-                            ValueRange typeDynDims = std::nullopt);
+                            ValueRange typeDynDims = {});
 
 /// Returns GenericOp that copies an n-D memref. Unlike the current
 /// implementation of memref::CopyOp, this op can further tile, lower to loops
diff --git a/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h b/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
index a1ce4e252c2f..da4c4ffb6d1b 100644
--- a/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
+++ b/mlir/include/mlir/Dialect/Tensor/Utils/Utils.h
@@ -30,7 +30,7 @@ namespace tensor {
 // for _static_ dimensions.
 PadOp createPadHighOp(RankedTensorType resType, Value source, Value pad,
                       bool nofold, Location loc, OpBuilder &builder,
-                      ValueRange dynOutDims = std::nullopt);
+                      ValueRange dynOutDims = {});
 
 // Creates dim ops for each dynamic dimension of the ranked tensor argument and
 // returns these as values.
diff --git a/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h b/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
index 66f49c787c19..96ccebcd5685 100644
--- a/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
+++ b/mlir/include/mlir/ExecutionEngine/ExecutionEngine.h
@@ -157,8 +157,7 @@ public:
 
   /// Invokes the function with the given name passing it the list of opaque
   /// pointers to the actual arguments.
-  llvm::Error invokePacked(StringRef name,
-                           MutableArrayRef<void *> args = std::nullopt);
+  llvm::Error invokePacked(StringRef name, MutableArrayRef<void *> args = {});
 
   /// Trait that defines how a given type is passed to the JIT code. This
   /// defaults to passing the address but can be specialized.
diff --git a/mlir/include/mlir/IR/BlockSupport.h b/mlir/include/mlir/IR/BlockSupport.h
index 41434269d555..f9fbef2f3753 100644
--- a/mlir/include/mlir/IR/BlockSupport.h
+++ b/mlir/include/mlir/IR/BlockSupport.h
@@ -106,7 +106,7 @@ class BlockRange final
           Block *, Block *, Block *> {
 public:
   using RangeBaseT::RangeBaseT;
-  BlockRange(ArrayRef<Block *> blocks = std::nullopt);
+  BlockRange(ArrayRef<Block *> blocks = {});
   BlockRange(SuccessorRange successors);
   template <typename Arg, typename = std::enable_if_t<std::is_constructible<
                               ArrayRef<Block *>, Arg>::value>>
diff --git a/mlir/include/mlir/IR/Builders.h b/mlir/include/mlir/IR/Builders.h
index d68dbdb1efee..ad59ea63a690 100644
--- a/mlir/include/mlir/IR/Builders.h
+++ b/mlir/include/mlir/IR/Builders.h
@@ -454,15 +454,14 @@ public:
   /// 'parent'. `locs` contains the locations of the inserted arguments, and
   /// should match the size of `argTypes`.
   Block *createBlock(Region *parent, Region::iterator insertPt = {},
-                     TypeRange argTypes = std::nullopt,
-                     ArrayRef<Location> locs = std::nullopt);
+                     TypeRange argTypes = {}, ArrayRef<Location> locs = {});
 
   /// Add new block with 'argTypes' arguments and set the insertion point to the
   /// end of it. The block is placed before 'insertBefore'. `locs` contains the
   /// locations of the inserted arguments, and should match the size of
   /// `argTypes`.
-  Block *createBlock(Block *insertBefore, TypeRange argTypes = std::nullopt,
-                     ArrayRef<Location> locs = std::nullopt);
+  Block *createBlock(Block *insertBefore, TypeRange argTypes = {},
+                     ArrayRef<Location> locs = {});
 
   //===--------------------------------------------------------------------===//
   // Operation Creation
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index 771de01fc8d5..d5663bcbf6a5 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -1200,7 +1200,7 @@ def Builtin_UnrankedMemRef : Builtin_Type<"UnrankedMemRef", "unranked_memref", [
     using ShapedType::Trait<UnrankedMemRefType>::getDimSize;
     using ShapedType::Trait<UnrankedMemRefType>::getDynamicDimIndex;
 
-    ArrayRef<int64_t> getShape() const { return std::nullopt; }
+    ArrayRef<int64_t> getShape() const { return {}; }
 
     /// [deprecated] Returns the memory space in old raw integer representation.
     /// New `Attribute getMemorySpace()` method should be used instead.
@@ -1259,7 +1259,7 @@ def Builtin_UnrankedTensor : Builtin_Type<"UnrankedTensor", "unranked_tensor", [
     using ShapedType::Trait<UnrankedTensorType>::getDimSize;
     using ShapedType::Trait<UnrankedTensorType>::getDynamicDimIndex;
 
-    ArrayRef<int64_t> getShape() const { return std::nullopt; }
+    ArrayRef<int64_t> getShape() const { return {}; }
   }];
   let skipDefaultBuilders = 1;
   let genVerifyDecl = 1;
diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h
index 10cfe851765d..b6659a7f915c 100644
--- a/mlir/include/mlir/IR/PatternMatch.h
+++ b/mlir/include/mlir/IR/PatternMatch.h
@@ -520,7 +520,7 @@ public:
   /// unreachable operations.
   virtual void inlineBlockBefore(Block *source, Block *dest,
                                  Block::iterator before,
-                                 ValueRange argValues = std::nullopt);
+                                 ValueRange argValues = {});
 
   /// Inline the operations of block 'source' before the operation 'op'. The
   /// source block will be deleted and must have no uses. 'argValues' is used to
@@ -529,7 +529,7 @@ public:
   /// The source block must have no successors. Otherwise, the resulting IR
   /// would have unreachable operations.
   void inlineBlockBefore(Block *source, Operation *op,
-                         ValueRange argValues = std::nullopt);
+                         ValueRange argValues = {});
 
   /// Inline the operations of block 'source' into the end of block 'dest'. The
   /// source block will be deleted and must have no uses. 'argValues' is used to
@@ -537,8 +537,7 @@ public:
   ///
   /// The dest block must have no successors. Otherwise, the resulting IR would
   /// have unreachable operation.
-  void mergeBlocks(Block *source, Block *dest,
-                   ValueRange argValues = std::nullopt);
+  void mergeBlocks(Block *source, Block *dest, ValueRange argValues = {});
 
   /// Split the operations starting at "before" (inclusive) out of the given
   /// block into a new block, and return it.
diff --git a/mlir/include/mlir/IR/Region.h b/mlir/include/mlir/IR/Region.h
index 22cb7037772d..1fcb31675023 100644
--- a/mlir/include/mlir/IR/Region.h
+++ b/mlir/include/mlir/IR/Region.h
@@ -353,7 +353,7 @@ class RegionRange
 public:
   using RangeBaseT::RangeBaseT;
 
-  RegionRange(MutableArrayRef<Region> regions = std::nullopt);
+  RegionRange(MutableArrayRef<Region> regions = {});
 
   template <typename Arg, typename = std::enable_if_t<std::is_constructible<
                               ArrayRef<std::unique_ptr<Region>>, Arg>::value>>
diff --git a/mlir/include/mlir/IR/SymbolTable.h b/mlir/include/mlir/IR/SymbolTable.h
index 557a2ba85cd6..e4622354b898 100644
--- a/mlir/include/mlir/IR/SymbolTable.h
+++ b/mlir/include/mlir/IR/SymbolTable.h
@@ -411,7 +411,8 @@ public:
   /// Return the users of the provided symbol operation.
   ArrayRef<Operation *> getUsers(Operation *symbol) const {
     auto it = symbolToUsers.find(symbol);
-    return it != symbolToUsers.end() ? it->second.getArrayRef() : std::nullopt;
+    return it != symbolToUsers.end() ? it->second.getArrayRef()
+                                     : ArrayRef<Operation *>();
   }
 
   /// Return true if the given symbol has no uses.
diff --git a/mlir/include/mlir/IR/TypeRange.h b/mlir/include/mlir/IR/TypeRange.h
index e098370ae6e5..c6cbf3461bcd 100644
--- a/mlir/include/mlir/IR/TypeRange.h
+++ b/mlir/include/mlir/IR/TypeRange.h
@@ -37,7 +37,7 @@ class TypeRange : public llvm::detail::indexed_accessor_range_base<
                       Type, Type, Type> {
 public:
   using RangeBaseT::RangeBaseT;
-  TypeRange(ArrayRef<Type> types = std::nullopt);
+  TypeRange(ArrayRef<Type> types = {});
   explicit TypeRange(OperandRange values);
   explicit TypeRange(ResultRange values);
   explicit TypeRange(ValueRange values);
diff --git a/mlir/include/mlir/IR/ValueRange.h b/mlir/include/mlir/IR/ValueRange.h
index 0c33e2b596b9..f04ed0544c0f 100644
--- a/mlir/include/mlir/IR/ValueRange.h
+++ b/mlir/include/mlir/IR/ValueRange.h
@@ -126,7 +126,7 @@ public:
   /// and range length. `operandSegments` is an optional set of operand segments
   /// to be updated when mutating the operand list.
   MutableOperandRange(Operation *owner, unsigned start, unsigned length,
-                      ArrayRef<OperandSegment> operandSegments = std::nullopt);
+                      ArrayRef<OperandSegment> operandSegments = {});
   MutableOperandRange(Operation *owner);
 
   /// Construct a new mutable range for the given OpOperand.
@@ -409,7 +409,7 @@ public:
       : ValueRange(ResultRange(values)) {}
   ValueRange(ArrayRef<BlockArgument> values)
       : ValueRange(ArrayRef<Value>(values.data(), values.size())) {}
-  ValueRange(ArrayRef<Value> values = std::nullopt);
+  ValueRange(ArrayRef<Value> values = {});
   ValueRange(OperandRange values);
   ValueRange(ResultRange values);
 
diff --git a/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h b/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
index 4c6e3cd9ce6f..d6c431206e06 100644
--- a/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
+++ b/mlir/include/mlir/Rewrite/FrozenRewritePatternSet.h
@@ -47,10 +47,9 @@ public:
   /// `RewritePatternSet::addWithLabel`. Debug names may be empty, but patterns
   /// created with `RewritePattern::create` have their default debug name set to
   /// their type name.
-  FrozenRewritePatternSet(
-      RewritePatternSet &&patterns,
-      ArrayRef<std::string> disabledPatternLabels = std::nullopt,
-      ArrayRef<std::string> enabledPatternLabels = std::nullopt);
+  FrozenRewritePatternSet(RewritePatternSet &&patterns,
+                          ArrayRef<std::string> disabledPatternLabels = {},
+                          ArrayRef<std::string> enabledPatternLabels = {});
 
   /// Return the op specific native patterns held by this list.
   const OpSpecificNativePatternListT &getOpSpecificNativePatterns() const {
diff --git a/mlir/include/mlir/Tools/PDLL/AST/Types.h b/mlir/include/mlir/Tools/PDLL/AST/Types.h
index 57161db5fdba..538ea7c61b44 100644
--- a/mlir/include/mlir/Tools/PDLL/AST/Types.h
+++ b/mlir/include/mlir/Tools/PDLL/AST/Types.h
@@ -226,8 +226,7 @@ public:
   /// Return an instance of the Tuple type.
   static TupleType get(Context &context, ArrayRef<Type> elementTypes,
                        ArrayRef<StringRef> elementNames);
-  static TupleType get(Context &context,
-                       ArrayRef<Type> elementTypes = std::nullopt);
+  static TupleType get(Context &context, ArrayRef<Type> elementTypes = {});
 
   /// Return the element types of this tuple.
   ArrayRef<Type> getElementTypes() const;
diff --git a/mlir/include/mlir/Transforms/DialectConversion.h b/mlir/include/mlir/Transforms/DialectConversion.h
index e7d05c3ce1ad..5a5f116073a9 100644
--- a/mlir/include/mlir/Transforms/DialectConversion.h
+++ b/mlir/include/mlir/Transforms/DialectConversion.h
@@ -826,7 +826,7 @@ public:
 
   /// PatternRewriter hook for inlining the ops of a block into another block.
   void inlineBlockBefore(Block *source, Block *dest, Block::iterator before,
-                         ValueRange argValues = std::nullopt) override;
+                         ValueRange argValues = {}) override;
   using PatternRewriter::inlineBlockBefore;
 
   /// PatternRewriter hook for updating the given operation in-place.
diff --git a/mlir/include/mlir/Transforms/Passes.h b/mlir/include/mlir/Transforms/Passes.h
index 41f208216374..9cd2ef34e15e 100644
--- a/mlir/include/mlir/Transforms/Passes.h
+++ b/mlir/include/mlir/Transforms/Passes.h
@@ -62,8 +62,8 @@ std::unique_ptr<Pass> createCanonicalizerPass();
 /// set to their type name.
 std::unique_ptr<Pass>
 createCanonicalizerPass(const GreedyRewriteConfig &config,
-                        ArrayRef<std::string> disabledPatterns = std::nullopt,
-                        ArrayRef<std::string> enabledPatterns = std::nullopt);
+                        ArrayRef<std::string> disabledPatterns = {},
+                        ArrayRef<std::string> enabledPatterns = {});
 
 /// Creates a pass to perform control-flow sinking.
 std::unique_ptr<Pass> createControlFlowSinkPass();
diff --git a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
index a64dc7f74a19..695d43b04cff 100644
--- a/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/DecomposeMemRefs.cpp
@@ -55,7 +55,7 @@ static bool isInsideLaunch(Operation *op) {
 static std::tuple<Value, OpFoldResult, SmallVector<OpFoldResult>>
 getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
                         ArrayRef<OpFoldResult> subOffsets,
-                        ArrayRef<OpFoldResult> subStrides = std::nullopt) {
+                        ArrayRef<OpFoldResult> subStrides = {}) {
   auto sourceType = cast<MemRefType>(source.getType());
   auto sourceRank = static_cast<unsigned>(sourceType.getRank());
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
index 8767b1c3ffc5..081a89d0b5dc 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
+++ b/mlir/lib/Dialect/LLVMIR/IR/TypeDetail.h
@@ -69,7 +69,7 @@ public:
   class Key {
   public:
     /// Constructs a key for an identified struct.
-    Key(StringRef name, bool opaque, ArrayRef<Type> types = std::nullopt)
+    Key(StringRef name, bool opaque, ArrayRef<Type> types = {})
         : types(types), name(name), identified(true), packed(false),
           opaque(opaque) {}
     /// Constructs a key for a literal struct.
diff --git a/mlir/lib/Pass/PassStatistics.cpp b/mlir/lib/Pass/PassStatistics.cpp
index 779dcfe7b666..01191aa82444 100644
--- a/mlir/lib/Pass/PassStatistics.cpp
+++ b/mlir/lib/Pass/PassStatistics.cpp
@@ -27,7 +27,7 @@ struct Statistic {
 
 /// Utility to print a pass entry in the statistics output.
 static void printPassEntry(raw_ostream &os, unsigned indent, StringRef pass,
-                           MutableArrayRef<Statistic> stats = std::nullopt) {
+                           MutableArrayRef<Statistic> stats = {}) {
   os.indent(indent) << pass << "\n";
   if (stats.empty())
     return;
diff --git a/mlir/unittests/IR/OperationSupportTest.cpp b/mlir/unittests/IR/OperationSupportTest.cpp
index 18ee9d71cb9f..6ea74a988ca8 100644
--- a/mlir/unittests/IR/OperationSupportTest.cpp
+++ b/mlir/unittests/IR/OperationSupportTest.cpp
@@ -18,9 +18,8 @@
 using namespace mlir;
 using namespace mlir::detail;
 
-static Operation *createOp(MLIRContext *context,
-                           ArrayRef<Value> operands = std::nullopt,
-                           ArrayRef<Type> resultTypes = std::nullopt,
+static Operation *createOp(MLIRContext *context, ArrayRef<Value> operands = {},
+                           ArrayRef<Type> resultTypes = {},
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
   return Operation::create(
diff --git a/mlir/unittests/IR/ValueTest.cpp b/mlir/unittests/IR/ValueTest.cpp
index 1a84b7ca13df..58678224780b 100644
--- a/mlir/unittests/IR/ValueTest.cpp
+++ b/mlir/unittests/IR/ValueTest.cpp
@@ -16,9 +16,8 @@
 
 using namespace mlir;
 
-static Operation *createOp(MLIRContext *context,
-                           ArrayRef<Value> operands = std::nullopt,
-                           ArrayRef<Type> resultTypes = std::nullopt,
+static Operation *createOp(MLIRContext *context, ArrayRef<Value> operands = {},
+                           ArrayRef<Type> resultTypes = {},
                            unsigned int numRegions = 0) {
   context->allowUnregisteredDialects();
   return Operation::create(
-- 
cgit v1.2.3


From 9524bfb27020d31b9474f595b7c0e5d2e1ac65f5 Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 20 Jun 2025 10:48:04 -0500
Subject: [lldb] Add Model Context Protocol (MCP) support to LLDB (#143628)

This PR adds an MCP (Model Context Protocol ) server to LLDB. For
motivation and background, please refer to the corresponding RFC:
https://discourse.llvm.org/t/rfc-adding-mcp-support-to-lldb/86798

I implemented this as a new kind of plugin. The idea is that we could
support multiple protocol servers (e.g. if we want to support DAP from
within LLDB). This also introduces a corresponding top-level command
(`protocol-server`) with two subcommands to `start` and `stop` the
server.

```
(lldb) protocol-server start MCP tcp://localhost:1234
MCP server started with connection listeners: connection://[::1]:1234, connection://[127.0.0.1]:1234
```

The MCP sever supports one tool (`lldb_command`) which executes a
command, but can easily be extended with more commands.
---
 lldb/cmake/modules/LLDBConfig.cmake                |   1 +
 lldb/include/lldb/Core/Debugger.h                  |   6 +
 lldb/include/lldb/Core/PluginManager.h             |  11 +
 lldb/include/lldb/Core/ProtocolServer.h            |  39 +++
 .../lldb/Interpreter/CommandOptionArgumentTable.h  |   1 +
 lldb/include/lldb/lldb-enumerations.h              |   1 +
 lldb/include/lldb/lldb-forward.h                   |   3 +-
 lldb/include/lldb/lldb-private-interfaces.h        |   2 +
 lldb/source/Commands/CMakeLists.txt                |   1 +
 .../Commands/CommandObjectProtocolServer.cpp       | 176 +++++++++++
 lldb/source/Commands/CommandObjectProtocolServer.h |  25 ++
 lldb/source/Core/CMakeLists.txt                    |   1 +
 lldb/source/Core/Debugger.cpp                      |  24 ++
 lldb/source/Core/PluginManager.cpp                 |  32 ++
 lldb/source/Core/ProtocolServer.cpp                |  21 ++
 lldb/source/Interpreter/CommandInterpreter.cpp     |   2 +
 lldb/source/Plugins/CMakeLists.txt                 |   4 +
 lldb/source/Plugins/Protocol/CMakeLists.txt        |   1 +
 lldb/source/Plugins/Protocol/MCP/CMakeLists.txt    |  13 +
 lldb/source/Plugins/Protocol/MCP/MCPError.cpp      |  34 +++
 lldb/source/Plugins/Protocol/MCP/MCPError.h        |  33 +++
 lldb/source/Plugins/Protocol/MCP/Protocol.cpp      | 214 ++++++++++++++
 lldb/source/Plugins/Protocol/MCP/Protocol.h        | 128 ++++++++
 .../Plugins/Protocol/MCP/ProtocolServerMCP.cpp     | 327 +++++++++++++++++++++
 .../Plugins/Protocol/MCP/ProtocolServerMCP.h       | 100 +++++++
 lldb/source/Plugins/Protocol/MCP/Tool.cpp          |  81 +++++
 lldb/source/Plugins/Protocol/MCP/Tool.h            |  56 ++++
 lldb/unittests/CMakeLists.txt                      |   4 +
 lldb/unittests/DAP/ProtocolTypesTest.cpp           |  38 +--
 lldb/unittests/Protocol/CMakeLists.txt             |  12 +
 lldb/unittests/Protocol/ProtocolMCPServerTest.cpp  | 291 ++++++++++++++++++
 lldb/unittests/Protocol/ProtocolMCPTest.cpp        | 135 +++++++++
 lldb/unittests/TestingSupport/TestUtilities.h      |   9 +
 33 files changed, 1803 insertions(+), 23 deletions(-)
 create mode 100644 lldb/include/lldb/Core/ProtocolServer.h
 create mode 100644 lldb/source/Commands/CommandObjectProtocolServer.cpp
 create mode 100644 lldb/source/Commands/CommandObjectProtocolServer.h
 create mode 100644 lldb/source/Core/ProtocolServer.cpp
 create mode 100644 lldb/source/Plugins/Protocol/CMakeLists.txt
 create mode 100644 lldb/source/Plugins/Protocol/MCP/CMakeLists.txt
 create mode 100644 lldb/source/Plugins/Protocol/MCP/MCPError.cpp
 create mode 100644 lldb/source/Plugins/Protocol/MCP/MCPError.h
 create mode 100644 lldb/source/Plugins/Protocol/MCP/Protocol.cpp
 create mode 100644 lldb/source/Plugins/Protocol/MCP/Protocol.h
 create mode 100644 lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
 create mode 100644 lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
 create mode 100644 lldb/source/Plugins/Protocol/MCP/Tool.cpp
 create mode 100644 lldb/source/Plugins/Protocol/MCP/Tool.h
 create mode 100644 lldb/unittests/Protocol/CMakeLists.txt
 create mode 100644 lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
 create mode 100644 lldb/unittests/Protocol/ProtocolMCPTest.cpp

diff --git a/lldb/cmake/modules/LLDBConfig.cmake b/lldb/cmake/modules/LLDBConfig.cmake
index 37b823feb584..8c30b6e09d2c 100644
--- a/lldb/cmake/modules/LLDBConfig.cmake
+++ b/lldb/cmake/modules/LLDBConfig.cmake
@@ -67,6 +67,7 @@ add_optional_dependency(LLDB_ENABLE_FBSDVMCORE "Enable libfbsdvmcore support in
 
 option(LLDB_USE_ENTITLEMENTS "When codesigning, use entitlements if available" ON)
 option(LLDB_BUILD_FRAMEWORK "Build LLDB.framework (Darwin only)" OFF)
+option(LLDB_ENABLE_PROTOCOL_SERVERS "Enable protocol servers (e.g. MCP) in LLDB" ON)
 option(LLDB_NO_INSTALL_DEFAULT_RPATH "Disable default RPATH settings in binaries" OFF)
 option(LLDB_USE_SYSTEM_DEBUGSERVER "Use the system's debugserver for testing (Darwin only)." OFF)
 option(LLDB_SKIP_STRIP "Whether to skip stripping of binaries when installing lldb." OFF)
diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h
index 2087ef2a1156..9f82466a8341 100644
--- a/lldb/include/lldb/Core/Debugger.h
+++ b/lldb/include/lldb/Core/Debugger.h
@@ -602,6 +602,10 @@ public:
   void FlushProcessOutput(Process &process, bool flush_stdout,
                           bool flush_stderr);
 
+  void AddProtocolServer(lldb::ProtocolServerSP protocol_server_sp);
+  void RemoveProtocolServer(lldb::ProtocolServerSP protocol_server_sp);
+  lldb::ProtocolServerSP GetProtocolServer(llvm::StringRef protocol) const;
+
   SourceManager::SourceFileCache &GetSourceFileCache() {
     return m_source_file_cache;
   }
@@ -772,6 +776,8 @@ protected:
   mutable std::mutex m_progress_reports_mutex;
   /// @}
 
+  llvm::SmallVector<lldb::ProtocolServerSP> m_protocol_servers;
+
   std::mutex m_destroy_callback_mutex;
   lldb::callback_token_t m_destroy_callback_next_token = 0;
   struct DestroyCallbackInfo {
diff --git a/lldb/include/lldb/Core/PluginManager.h b/lldb/include/lldb/Core/PluginManager.h
index 1d7c976f3c38..d1af25988e50 100644
--- a/lldb/include/lldb/Core/PluginManager.h
+++ b/lldb/include/lldb/Core/PluginManager.h
@@ -321,6 +321,17 @@ public:
   static void AutoCompleteProcessName(llvm::StringRef partial_name,
                                       CompletionRequest &request);
 
+  // Protocol
+  static bool RegisterPlugin(llvm::StringRef name, llvm::StringRef description,
+                             ProtocolServerCreateInstance create_callback);
+
+  static bool UnregisterPlugin(ProtocolServerCreateInstance create_callback);
+
+  static llvm::StringRef GetProtocolServerPluginNameAtIndex(uint32_t idx);
+
+  static ProtocolServerCreateInstance
+  GetProtocolCreateCallbackForPluginName(llvm::StringRef name);
+
   // Register Type Provider
   static bool RegisterPlugin(llvm::StringRef name, llvm::StringRef description,
                              RegisterTypeBuilderCreateInstance create_callback);
diff --git a/lldb/include/lldb/Core/ProtocolServer.h b/lldb/include/lldb/Core/ProtocolServer.h
new file mode 100644
index 000000000000..fafe46090432
--- /dev/null
+++ b/lldb/include/lldb/Core/ProtocolServer.h
@@ -0,0 +1,39 @@
+//===-- ProtocolServer.h --------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_CORE_PROTOCOLSERVER_H
+#define LLDB_CORE_PROTOCOLSERVER_H
+
+#include "lldb/Core/PluginInterface.h"
+#include "lldb/Host/Socket.h"
+#include "lldb/lldb-private-interfaces.h"
+
+namespace lldb_private {
+
+class ProtocolServer : public PluginInterface {
+public:
+  ProtocolServer() = default;
+  virtual ~ProtocolServer() = default;
+
+  static lldb::ProtocolServerSP Create(llvm::StringRef name,
+                                       Debugger &debugger);
+
+  struct Connection {
+    Socket::SocketProtocol protocol;
+    std::string name;
+  };
+
+  virtual llvm::Error Start(Connection connection) = 0;
+  virtual llvm::Error Stop() = 0;
+
+  virtual Socket *GetSocket() const = 0;
+};
+
+} // namespace lldb_private
+
+#endif
diff --git a/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h b/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
index 8535dfcf46da..4face717531b 100644
--- a/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
+++ b/lldb/include/lldb/Interpreter/CommandOptionArgumentTable.h
@@ -315,6 +315,7 @@ static constexpr CommandObject::ArgumentTableEntry g_argument_table[] = {
     { lldb::eArgTypeCPUName, "cpu-name", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "The name of a CPU." },
     { lldb::eArgTypeCPUFeatures, "cpu-features", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "The CPU feature string." },
     { lldb::eArgTypeManagedPlugin, "managed-plugin", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "Plugins managed by the PluginManager" },
+    { lldb::eArgTypeProtocol, "protocol", lldb::CompletionType::eNoCompletion, {}, { nullptr, false }, "The name of the protocol." },
     // clang-format on
 };
 
diff --git a/lldb/include/lldb/lldb-enumerations.h b/lldb/include/lldb/lldb-enumerations.h
index eeb7299a354e..69e8671b6e21 100644
--- a/lldb/include/lldb/lldb-enumerations.h
+++ b/lldb/include/lldb/lldb-enumerations.h
@@ -664,6 +664,7 @@ enum CommandArgumentType {
   eArgTypeCPUName,
   eArgTypeCPUFeatures,
   eArgTypeManagedPlugin,
+  eArgTypeProtocol,
   eArgTypeLastArg // Always keep this entry as the last entry in this
                   // enumeration!!
 };
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index c664d1398f74..558818e8e230 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -164,13 +164,13 @@ class PersistentExpressionState;
 class Platform;
 class Process;
 class ProcessAttachInfo;
-class ProcessLaunchInfo;
 class ProcessInfo;
 class ProcessInstanceInfo;
 class ProcessInstanceInfoMatch;
 class ProcessLaunchInfo;
 class ProcessModID;
 class Property;
+class ProtocolServer;
 class Queue;
 class QueueImpl;
 class QueueItem;
@@ -391,6 +391,7 @@ typedef std::shared_ptr<lldb_private::Platform> PlatformSP;
 typedef std::shared_ptr<lldb_private::Process> ProcessSP;
 typedef std::shared_ptr<lldb_private::ProcessAttachInfo> ProcessAttachInfoSP;
 typedef std::shared_ptr<lldb_private::ProcessLaunchInfo> ProcessLaunchInfoSP;
+typedef std::shared_ptr<lldb_private::ProtocolServer> ProtocolServerSP;
 typedef std::weak_ptr<lldb_private::Process> ProcessWP;
 typedef std::shared_ptr<lldb_private::RegisterCheckpoint> RegisterCheckpointSP;
 typedef std::shared_ptr<lldb_private::RegisterContext> RegisterContextSP;
diff --git a/lldb/include/lldb/lldb-private-interfaces.h b/lldb/include/lldb/lldb-private-interfaces.h
index d366dbd1d783..34eaaa8e581e 100644
--- a/lldb/include/lldb/lldb-private-interfaces.h
+++ b/lldb/include/lldb/lldb-private-interfaces.h
@@ -81,6 +81,8 @@ typedef lldb::PlatformSP (*PlatformCreateInstance)(bool force,
 typedef lldb::ProcessSP (*ProcessCreateInstance)(
     lldb::TargetSP target_sp, lldb::ListenerSP listener_sp,
     const FileSpec *crash_file_path, bool can_connect);
+typedef lldb::ProtocolServerSP (*ProtocolServerCreateInstance)(
+    Debugger &debugger);
 typedef lldb::RegisterTypeBuilderSP (*RegisterTypeBuilderCreateInstance)(
     Target &target);
 typedef lldb::ScriptInterpreterSP (*ScriptInterpreterCreateInstance)(
diff --git a/lldb/source/Commands/CMakeLists.txt b/lldb/source/Commands/CMakeLists.txt
index 1ea51acec5f1..69e4c45f0b8e 100644
--- a/lldb/source/Commands/CMakeLists.txt
+++ b/lldb/source/Commands/CMakeLists.txt
@@ -23,6 +23,7 @@ add_lldb_library(lldbCommands NO_PLUGIN_DEPENDENCIES
   CommandObjectPlatform.cpp
   CommandObjectPlugin.cpp
   CommandObjectProcess.cpp
+  CommandObjectProtocolServer.cpp
   CommandObjectQuit.cpp
   CommandObjectRegexCommand.cpp
   CommandObjectRegister.cpp
diff --git a/lldb/source/Commands/CommandObjectProtocolServer.cpp b/lldb/source/Commands/CommandObjectProtocolServer.cpp
new file mode 100644
index 000000000000..420fc5fdddad
--- /dev/null
+++ b/lldb/source/Commands/CommandObjectProtocolServer.cpp
@@ -0,0 +1,176 @@
+//===-- CommandObjectProtocolServer.cpp
+//----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "CommandObjectProtocolServer.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Core/ProtocolServer.h"
+#include "lldb/Host/Socket.h"
+#include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/CommandReturnObject.h"
+#include "lldb/Utility/UriParser.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Support/FormatAdapters.h"
+
+using namespace llvm;
+using namespace lldb;
+using namespace lldb_private;
+
+#define LLDB_OPTIONS_mcp
+#include "CommandOptions.inc"
+
+static std::vector<llvm::StringRef> GetSupportedProtocols() {
+  std::vector<llvm::StringRef> supported_protocols;
+  size_t i = 0;
+
+  for (llvm::StringRef protocol_name =
+           PluginManager::GetProtocolServerPluginNameAtIndex(i++);
+       !protocol_name.empty();
+       protocol_name = PluginManager::GetProtocolServerPluginNameAtIndex(i++)) {
+    supported_protocols.push_back(protocol_name);
+  }
+
+  return supported_protocols;
+}
+
+class CommandObjectProtocolServerStart : public CommandObjectParsed {
+public:
+  CommandObjectProtocolServerStart(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "protocol-server start",
+                            "start protocol server",
+                            "protocol-server start <protocol> <connection>") {
+    AddSimpleArgumentList(lldb::eArgTypeProtocol, eArgRepeatPlain);
+    AddSimpleArgumentList(lldb::eArgTypeConnectURL, eArgRepeatPlain);
+  }
+
+  ~CommandObjectProtocolServerStart() override = default;
+
+protected:
+  void DoExecute(Args &args, CommandReturnObject &result) override {
+    if (args.GetArgumentCount() < 1) {
+      result.AppendError("no protocol specified");
+      return;
+    }
+
+    llvm::StringRef protocol = args.GetArgumentAtIndex(0);
+    std::vector<llvm::StringRef> supported_protocols = GetSupportedProtocols();
+    if (llvm::find(supported_protocols, protocol) ==
+        supported_protocols.end()) {
+      result.AppendErrorWithFormatv(
+          "unsupported protocol: {0}. Supported protocols are: {1}", protocol,
+          llvm::join(GetSupportedProtocols(), ", "));
+      return;
+    }
+
+    if (args.GetArgumentCount() < 2) {
+      result.AppendError("no connection specified");
+      return;
+    }
+    llvm::StringRef connection_uri = args.GetArgumentAtIndex(1);
+
+    ProtocolServerSP server_sp = GetDebugger().GetProtocolServer(protocol);
+    if (!server_sp)
+      server_sp = ProtocolServer::Create(protocol, GetDebugger());
+
+    const char *connection_error =
+        "unsupported connection specifier, expected 'accept:///path' or "
+        "'listen://[host]:port', got '{0}'.";
+    auto uri = lldb_private::URI::Parse(connection_uri);
+    if (!uri) {
+      result.AppendErrorWithFormatv(connection_error, connection_uri);
+      return;
+    }
+
+    std::optional<Socket::ProtocolModePair> protocol_and_mode =
+        Socket::GetProtocolAndMode(uri->scheme);
+    if (!protocol_and_mode || protocol_and_mode->second != Socket::ModeAccept) {
+      result.AppendErrorWithFormatv(connection_error, connection_uri);
+      return;
+    }
+
+    ProtocolServer::Connection connection;
+    connection.protocol = protocol_and_mode->first;
+    connection.name =
+        formatv("[{0}]:{1}", uri->hostname.empty() ? "0.0.0.0" : uri->hostname,
+                uri->port.value_or(0));
+
+    if (llvm::Error error = server_sp->Start(connection)) {
+      result.AppendErrorWithFormatv("{0}", llvm::fmt_consume(std::move(error)));
+      return;
+    }
+
+    GetDebugger().AddProtocolServer(server_sp);
+
+    if (Socket *socket = server_sp->GetSocket()) {
+      std::string address =
+          llvm::join(socket->GetListeningConnectionURI(), ", ");
+      result.AppendMessageWithFormatv(
+          "{0} server started with connection listeners: {1}", protocol,
+          address);
+    }
+  }
+};
+
+class CommandObjectProtocolServerStop : public CommandObjectParsed {
+public:
+  CommandObjectProtocolServerStop(CommandInterpreter &interpreter)
+      : CommandObjectParsed(interpreter, "protocol-server stop",
+                            "stop protocol server",
+                            "protocol-server stop <protocol>") {
+    AddSimpleArgumentList(lldb::eArgTypeProtocol, eArgRepeatPlain);
+  }
+
+  ~CommandObjectProtocolServerStop() override = default;
+
+protected:
+  void DoExecute(Args &args, CommandReturnObject &result) override {
+    if (args.GetArgumentCount() < 1) {
+      result.AppendError("no protocol specified");
+      return;
+    }
+
+    llvm::StringRef protocol = args.GetArgumentAtIndex(0);
+    std::vector<llvm::StringRef> supported_protocols = GetSupportedProtocols();
+    if (llvm::find(supported_protocols, protocol) ==
+        supported_protocols.end()) {
+      result.AppendErrorWithFormatv(
+          "unsupported protocol: {0}. Supported protocols are: {1}", protocol,
+          llvm::join(GetSupportedProtocols(), ", "));
+      return;
+    }
+
+    Debugger &debugger = GetDebugger();
+
+    ProtocolServerSP server_sp = debugger.GetProtocolServer(protocol);
+    if (!server_sp) {
+      result.AppendError(
+          llvm::formatv("no {0} protocol server running", protocol).str());
+      return;
+    }
+
+    if (llvm::Error error = server_sp->Stop()) {
+      result.AppendErrorWithFormatv("{0}", llvm::fmt_consume(std::move(error)));
+      return;
+    }
+
+    debugger.RemoveProtocolServer(server_sp);
+  }
+};
+
+CommandObjectProtocolServer::CommandObjectProtocolServer(
+    CommandInterpreter &interpreter)
+    : CommandObjectMultiword(interpreter, "protocol-server",
+                             "Start and stop a protocol server.",
+                             "protocol-server") {
+  LoadSubCommand("start", CommandObjectSP(new CommandObjectProtocolServerStart(
+                              interpreter)));
+  LoadSubCommand("stop", CommandObjectSP(
+                             new CommandObjectProtocolServerStop(interpreter)));
+}
+
+CommandObjectProtocolServer::~CommandObjectProtocolServer() = default;
diff --git a/lldb/source/Commands/CommandObjectProtocolServer.h b/lldb/source/Commands/CommandObjectProtocolServer.h
new file mode 100644
index 000000000000..3591216b014c
--- /dev/null
+++ b/lldb/source/Commands/CommandObjectProtocolServer.h
@@ -0,0 +1,25 @@
+//===-- CommandObjectProtocolServer.h
+//------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_SOURCE_COMMANDS_COMMANDOBJECTPROTOCOLSERVER_H
+#define LLDB_SOURCE_COMMANDS_COMMANDOBJECTPROTOCOLSERVER_H
+
+#include "lldb/Interpreter/CommandObjectMultiword.h"
+
+namespace lldb_private {
+
+class CommandObjectProtocolServer : public CommandObjectMultiword {
+public:
+  CommandObjectProtocolServer(CommandInterpreter &interpreter);
+  ~CommandObjectProtocolServer() override;
+};
+
+} // namespace lldb_private
+
+#endif // LLDB_SOURCE_COMMANDS_COMMANDOBJECTMCP_H
diff --git a/lldb/source/Core/CMakeLists.txt b/lldb/source/Core/CMakeLists.txt
index d6b75bca7f2d..df35bd5c025f 100644
--- a/lldb/source/Core/CMakeLists.txt
+++ b/lldb/source/Core/CMakeLists.txt
@@ -46,6 +46,7 @@ add_lldb_library(lldbCore NO_PLUGIN_DEPENDENCIES
   Opcode.cpp
   PluginManager.cpp
   Progress.cpp
+  ProtocolServer.cpp
   Statusline.cpp
   RichManglingContext.cpp
   SearchFilter.cpp
diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp
index c9935f2d745f..33d1053fd8a6 100644
--- a/lldb/source/Core/Debugger.cpp
+++ b/lldb/source/Core/Debugger.cpp
@@ -16,6 +16,7 @@
 #include "lldb/Core/ModuleSpec.h"
 #include "lldb/Core/PluginManager.h"
 #include "lldb/Core/Progress.h"
+#include "lldb/Core/ProtocolServer.h"
 #include "lldb/Core/StreamAsynchronousIO.h"
 #include "lldb/Core/Telemetry.h"
 #include "lldb/DataFormatters/DataVisualization.h"
@@ -2375,3 +2376,26 @@ llvm::ThreadPoolInterface &Debugger::GetThreadPool() {
          "Debugger::GetThreadPool called before Debugger::Initialize");
   return *g_thread_pool;
 }
+
+void Debugger::AddProtocolServer(lldb::ProtocolServerSP protocol_server_sp) {
+  assert(protocol_server_sp &&
+         GetProtocolServer(protocol_server_sp->GetPluginName()) == nullptr);
+  m_protocol_servers.push_back(protocol_server_sp);
+}
+
+void Debugger::RemoveProtocolServer(lldb::ProtocolServerSP protocol_server_sp) {
+  auto it = llvm::find(m_protocol_servers, protocol_server_sp);
+  if (it != m_protocol_servers.end())
+    m_protocol_servers.erase(it);
+}
+
+lldb::ProtocolServerSP
+Debugger::GetProtocolServer(llvm::StringRef protocol) const {
+  for (ProtocolServerSP protocol_server_sp : m_protocol_servers) {
+    if (!protocol_server_sp)
+      continue;
+    if (protocol_server_sp->GetPluginName() == protocol)
+      return protocol_server_sp;
+  }
+  return nullptr;
+}
diff --git a/lldb/source/Core/PluginManager.cpp b/lldb/source/Core/PluginManager.cpp
index dfa865929b64..dc0731c04eef 100644
--- a/lldb/source/Core/PluginManager.cpp
+++ b/lldb/source/Core/PluginManager.cpp
@@ -1152,6 +1152,38 @@ void PluginManager::AutoCompleteProcessName(llvm::StringRef name,
   }
 }
 
+#pragma mark ProtocolServer
+
+typedef PluginInstance<ProtocolServerCreateInstance> ProtocolServerInstance;
+typedef PluginInstances<ProtocolServerInstance> ProtocolServerInstances;
+
+static ProtocolServerInstances &GetProtocolServerInstances() {
+  static ProtocolServerInstances g_instances;
+  return g_instances;
+}
+
+bool PluginManager::RegisterPlugin(
+    llvm::StringRef name, llvm::StringRef description,
+    ProtocolServerCreateInstance create_callback) {
+  return GetProtocolServerInstances().RegisterPlugin(name, description,
+                                                     create_callback);
+}
+
+bool PluginManager::UnregisterPlugin(
+    ProtocolServerCreateInstance create_callback) {
+  return GetProtocolServerInstances().UnregisterPlugin(create_callback);
+}
+
+llvm::StringRef
+PluginManager::GetProtocolServerPluginNameAtIndex(uint32_t idx) {
+  return GetProtocolServerInstances().GetNameAtIndex(idx);
+}
+
+ProtocolServerCreateInstance
+PluginManager::GetProtocolCreateCallbackForPluginName(llvm::StringRef name) {
+  return GetProtocolServerInstances().GetCallbackForName(name);
+}
+
 #pragma mark RegisterTypeBuilder
 
 struct RegisterTypeBuilderInstance
diff --git a/lldb/source/Core/ProtocolServer.cpp b/lldb/source/Core/ProtocolServer.cpp
new file mode 100644
index 000000000000..d57a047afa7b
--- /dev/null
+++ b/lldb/source/Core/ProtocolServer.cpp
@@ -0,0 +1,21 @@
+//===-- ProtocolServer.cpp ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lldb/Core/ProtocolServer.h"
+#include "lldb/Core/PluginManager.h"
+
+using namespace lldb_private;
+using namespace lldb;
+
+ProtocolServerSP ProtocolServer::Create(llvm::StringRef name,
+                                        Debugger &debugger) {
+  if (ProtocolServerCreateInstance create_callback =
+          PluginManager::GetProtocolCreateCallbackForPluginName(name))
+    return create_callback(debugger);
+  return nullptr;
+}
diff --git a/lldb/source/Interpreter/CommandInterpreter.cpp b/lldb/source/Interpreter/CommandInterpreter.cpp
index 4f9ae104dede..00c3472444d2 100644
--- a/lldb/source/Interpreter/CommandInterpreter.cpp
+++ b/lldb/source/Interpreter/CommandInterpreter.cpp
@@ -30,6 +30,7 @@
 #include "Commands/CommandObjectPlatform.h"
 #include "Commands/CommandObjectPlugin.h"
 #include "Commands/CommandObjectProcess.h"
+#include "Commands/CommandObjectProtocolServer.h"
 #include "Commands/CommandObjectQuit.h"
 #include "Commands/CommandObjectRegexCommand.h"
 #include "Commands/CommandObjectRegister.h"
@@ -574,6 +575,7 @@ void CommandInterpreter::LoadCommandDictionary() {
   REGISTER_COMMAND_OBJECT("platform", CommandObjectPlatform);
   REGISTER_COMMAND_OBJECT("plugin", CommandObjectPlugin);
   REGISTER_COMMAND_OBJECT("process", CommandObjectMultiwordProcess);
+  REGISTER_COMMAND_OBJECT("protocol-server", CommandObjectProtocolServer);
   REGISTER_COMMAND_OBJECT("quit", CommandObjectQuit);
   REGISTER_COMMAND_OBJECT("register", CommandObjectRegister);
   REGISTER_COMMAND_OBJECT("scripting", CommandObjectMultiwordScripting);
diff --git a/lldb/source/Plugins/CMakeLists.txt b/lldb/source/Plugins/CMakeLists.txt
index 854f589f45ae..08f444e7b15e 100644
--- a/lldb/source/Plugins/CMakeLists.txt
+++ b/lldb/source/Plugins/CMakeLists.txt
@@ -27,6 +27,10 @@ add_subdirectory(TraceExporter)
 add_subdirectory(TypeSystem)
 add_subdirectory(UnwindAssembly)
 
+if(LLDB_ENABLE_PROTOCOL_SERVERS)
+  add_subdirectory(Protocol)
+endif()
+
 set(LLDB_STRIPPED_PLUGINS)
 get_property(LLDB_ALL_PLUGINS GLOBAL PROPERTY LLDB_PLUGINS)
 
diff --git a/lldb/source/Plugins/Protocol/CMakeLists.txt b/lldb/source/Plugins/Protocol/CMakeLists.txt
new file mode 100644
index 000000000000..93b347d4cc9d
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(MCP)
diff --git a/lldb/source/Plugins/Protocol/MCP/CMakeLists.txt b/lldb/source/Plugins/Protocol/MCP/CMakeLists.txt
new file mode 100644
index 000000000000..db31a7a69cb3
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/CMakeLists.txt
@@ -0,0 +1,13 @@
+add_lldb_library(lldbPluginProtocolServerMCP PLUGIN
+  MCPError.cpp
+  Protocol.cpp
+  ProtocolServerMCP.cpp
+  Tool.cpp
+
+  LINK_COMPONENTS
+    Support
+
+  LINK_LIBS
+    lldbHost
+    lldbUtility
+)
diff --git a/lldb/source/Plugins/Protocol/MCP/MCPError.cpp b/lldb/source/Plugins/Protocol/MCP/MCPError.cpp
new file mode 100644
index 000000000000..5ed850066b65
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/MCPError.cpp
@@ -0,0 +1,34 @@
+//===-- MCPError.cpp ------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "MCPError.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/raw_ostream.h"
+#include <system_error>
+
+namespace lldb_private::mcp {
+
+char MCPError::ID;
+
+MCPError::MCPError(std::string message, int64_t error_code)
+    : m_message(message), m_error_code(error_code) {}
+
+void MCPError::log(llvm::raw_ostream &OS) const { OS << m_message; }
+
+std::error_code MCPError::convertToErrorCode() const {
+  return llvm::inconvertibleErrorCode();
+}
+
+protocol::Error MCPError::toProtcolError() const {
+  protocol::Error error;
+  error.error.code = m_error_code;
+  error.error.message = m_message;
+  return error;
+}
+
+} // namespace lldb_private::mcp
diff --git a/lldb/source/Plugins/Protocol/MCP/MCPError.h b/lldb/source/Plugins/Protocol/MCP/MCPError.h
new file mode 100644
index 000000000000..2a76a7b087e2
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/MCPError.h
@@ -0,0 +1,33 @@
+//===-- MCPError.h --------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Protocol.h"
+#include "llvm/Support/Error.h"
+#include <string>
+
+namespace lldb_private::mcp {
+
+class MCPError : public llvm::ErrorInfo<MCPError> {
+public:
+  static char ID;
+
+  MCPError(std::string message, int64_t error_code);
+
+  void log(llvm::raw_ostream &OS) const override;
+  std::error_code convertToErrorCode() const override;
+
+  const std::string &getMessage() const { return m_message; }
+
+  protocol::Error toProtcolError() const;
+
+private:
+  std::string m_message;
+  int64_t m_error_code;
+};
+
+} // namespace lldb_private::mcp
diff --git a/lldb/source/Plugins/Protocol/MCP/Protocol.cpp b/lldb/source/Plugins/Protocol/MCP/Protocol.cpp
new file mode 100644
index 000000000000..d66c931a0b28
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/Protocol.cpp
@@ -0,0 +1,214 @@
+//===- Protocol.cpp -------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Protocol.h"
+#include "llvm/Support/JSON.h"
+
+using namespace llvm;
+
+namespace lldb_private::mcp::protocol {
+
+static bool mapRaw(const json::Value &Params, StringLiteral Prop,
+                   std::optional<json::Value> &V, json::Path P) {
+  const auto *O = Params.getAsObject();
+  if (!O) {
+    P.report("expected object");
+    return false;
+  }
+  const json::Value *E = O->get(Prop);
+  if (E)
+    V = std::move(*E);
+  return true;
+}
+
+llvm::json::Value toJSON(const Request &R) {
+  json::Object Result{{"jsonrpc", "2.0"}, {"id", R.id}, {"method", R.method}};
+  if (R.params)
+    Result.insert({"params", R.params});
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &V, Request &R, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  if (!O || !O.map("id", R.id) || !O.map("method", R.method))
+    return false;
+  return mapRaw(V, "params", R.params, P);
+}
+
+llvm::json::Value toJSON(const ErrorInfo &EI) {
+  llvm::json::Object Result{{"code", EI.code}, {"message", EI.message}};
+  if (EI.data)
+    Result.insert({"data", EI.data});
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &V, ErrorInfo &EI, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("code", EI.code) && O.map("message", EI.message) &&
+         O.mapOptional("data", EI.data);
+}
+
+llvm::json::Value toJSON(const Error &E) {
+  return json::Object{{"jsonrpc", "2.0"}, {"id", E.id}, {"error", E.error}};
+}
+
+bool fromJSON(const llvm::json::Value &V, Error &E, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("id", E.id) && O.map("error", E.error);
+}
+
+llvm::json::Value toJSON(const Response &R) {
+  llvm::json::Object Result{{"jsonrpc", "2.0"}, {"id", R.id}};
+  if (R.result)
+    Result.insert({"result", R.result});
+  if (R.error)
+    Result.insert({"error", R.error});
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &V, Response &R, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  if (!O || !O.map("id", R.id) || !O.map("error", R.error))
+    return false;
+  return mapRaw(V, "result", R.result, P);
+}
+
+llvm::json::Value toJSON(const Notification &N) {
+  llvm::json::Object Result{{"jsonrpc", "2.0"}, {"method", N.method}};
+  if (N.params)
+    Result.insert({"params", N.params});
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &V, Notification &N, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  if (!O || !O.map("method", N.method))
+    return false;
+  auto *Obj = V.getAsObject();
+  if (!Obj)
+    return false;
+  if (auto *Params = Obj->get("params"))
+    N.params = *Params;
+  return true;
+}
+
+llvm::json::Value toJSON(const ToolCapability &TC) {
+  return llvm::json::Object{{"listChanged", TC.listChanged}};
+}
+
+bool fromJSON(const llvm::json::Value &V, ToolCapability &TC,
+              llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("listChanged", TC.listChanged);
+}
+
+llvm::json::Value toJSON(const Capabilities &C) {
+  return llvm::json::Object{{"tools", C.tools}};
+}
+
+bool fromJSON(const llvm::json::Value &V, Capabilities &C, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("tools", C.tools);
+}
+
+llvm::json::Value toJSON(const TextContent &TC) {
+  return llvm::json::Object{{"type", "text"}, {"text", TC.text}};
+}
+
+bool fromJSON(const llvm::json::Value &V, TextContent &TC, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("text", TC.text);
+}
+
+llvm::json::Value toJSON(const TextResult &TR) {
+  return llvm::json::Object{{"content", TR.content}, {"isError", TR.isError}};
+}
+
+bool fromJSON(const llvm::json::Value &V, TextResult &TR, llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("content", TR.content) && O.map("isError", TR.isError);
+}
+
+llvm::json::Value toJSON(const ToolDefinition &TD) {
+  llvm::json::Object Result{{"name", TD.name}};
+  if (TD.description)
+    Result.insert({"description", TD.description});
+  if (TD.inputSchema)
+    Result.insert({"inputSchema", TD.inputSchema});
+  return Result;
+}
+
+bool fromJSON(const llvm::json::Value &V, ToolDefinition &TD,
+              llvm::json::Path P) {
+
+  llvm::json::ObjectMapper O(V, P);
+  if (!O || !O.map("name", TD.name) ||
+      !O.mapOptional("description", TD.description))
+    return false;
+  return mapRaw(V, "inputSchema", TD.inputSchema, P);
+}
+
+llvm::json::Value toJSON(const Message &M) {
+  return std::visit([](auto &M) { return toJSON(M); }, M);
+}
+
+bool fromJSON(const llvm::json::Value &V, Message &M, llvm::json::Path P) {
+  const auto *O = V.getAsObject();
+  if (!O) {
+    P.report("expected object");
+    return false;
+  }
+
+  if (const json::Value *V = O->get("jsonrpc")) {
+    if (V->getAsString().value_or("") != "2.0") {
+      P.report("unsupported JSON RPC version");
+      return false;
+    }
+  } else {
+    P.report("not a valid JSON RPC message");
+    return false;
+  }
+
+  // A message without an ID is a Notification.
+  if (!O->get("id")) {
+    protocol::Notification N;
+    if (!fromJSON(V, N, P))
+      return false;
+    M = std::move(N);
+    return true;
+  }
+
+  if (O->get("error")) {
+    protocol::Error E;
+    if (!fromJSON(V, E, P))
+      return false;
+    M = std::move(E);
+    return true;
+  }
+
+  if (O->get("result")) {
+    protocol::Response R;
+    if (!fromJSON(V, R, P))
+      return false;
+    M = std::move(R);
+    return true;
+  }
+
+  if (O->get("method")) {
+    protocol::Request R;
+    if (!fromJSON(V, R, P))
+      return false;
+    M = std::move(R);
+    return true;
+  }
+
+  P.report("unrecognized message type");
+  return false;
+}
+
+} // namespace lldb_private::mcp::protocol
diff --git a/lldb/source/Plugins/Protocol/MCP/Protocol.h b/lldb/source/Plugins/Protocol/MCP/Protocol.h
new file mode 100644
index 000000000000..e31589940657
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/Protocol.h
@@ -0,0 +1,128 @@
+//===- Protocol.h ---------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains POD structs based on the MCP specification at
+// https://github.com/modelcontextprotocol/modelcontextprotocol/blob/main/schema/2024-11-05/schema.json
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_PROTOCOL_MCP_PROTOCOL_H
+#define LLDB_PLUGINS_PROTOCOL_MCP_PROTOCOL_H
+
+#include "llvm/Support/JSON.h"
+#include <optional>
+#include <string>
+#include <variant>
+
+namespace lldb_private::mcp::protocol {
+
+static llvm::StringLiteral kVersion = "2024-11-05";
+
+/// A request that expects a response.
+struct Request {
+  uint64_t id = 0;
+  std::string method;
+  std::optional<llvm::json::Value> params;
+};
+
+llvm::json::Value toJSON(const Request &);
+bool fromJSON(const llvm::json::Value &, Request &, llvm::json::Path);
+
+struct ErrorInfo {
+  int64_t code = 0;
+  std::string message;
+  std::optional<std::string> data;
+};
+
+llvm::json::Value toJSON(const ErrorInfo &);
+bool fromJSON(const llvm::json::Value &, ErrorInfo &, llvm::json::Path);
+
+struct Error {
+  uint64_t id = 0;
+  ErrorInfo error;
+};
+
+llvm::json::Value toJSON(const Error &);
+bool fromJSON(const llvm::json::Value &, Error &, llvm::json::Path);
+
+struct Response {
+  uint64_t id = 0;
+  std::optional<llvm::json::Value> result;
+  std::optional<ErrorInfo> error;
+};
+
+llvm::json::Value toJSON(const Response &);
+bool fromJSON(const llvm::json::Value &, Response &, llvm::json::Path);
+
+/// A notification which does not expect a response.
+struct Notification {
+  std::string method;
+  std::optional<llvm::json::Value> params;
+};
+
+llvm::json::Value toJSON(const Notification &);
+bool fromJSON(const llvm::json::Value &, Notification &, llvm::json::Path);
+
+struct ToolCapability {
+  /// Whether this server supports notifications for changes to the tool list.
+  bool listChanged = false;
+};
+
+llvm::json::Value toJSON(const ToolCapability &);
+bool fromJSON(const llvm::json::Value &, ToolCapability &, llvm::json::Path);
+
+/// Capabilities that a server may support. Known capabilities are defined here,
+/// in this schema, but this is not a closed set: any server can define its own,
+/// additional capabilities.
+struct Capabilities {
+  /// Present if the server offers any tools to call.
+  ToolCapability tools;
+};
+
+llvm::json::Value toJSON(const Capabilities &);
+bool fromJSON(const llvm::json::Value &, Capabilities &, llvm::json::Path);
+
+/// Text provided to or from an LLM.
+struct TextContent {
+  /// The text content of the message.
+  std::string text;
+};
+
+llvm::json::Value toJSON(const TextContent &);
+bool fromJSON(const llvm::json::Value &, TextContent &, llvm::json::Path);
+
+struct TextResult {
+  std::vector<TextContent> content;
+  bool isError = false;
+};
+
+llvm::json::Value toJSON(const TextResult &);
+bool fromJSON(const llvm::json::Value &, TextResult &, llvm::json::Path);
+
+struct ToolDefinition {
+  /// Unique identifier for the tool.
+  std::string name;
+
+  /// Human-readable description.
+  std::optional<std::string> description;
+
+  // JSON Schema for the tool's parameters.
+  std::optional<llvm::json::Value> inputSchema;
+};
+
+llvm::json::Value toJSON(const ToolDefinition &);
+bool fromJSON(const llvm::json::Value &, ToolDefinition &, llvm::json::Path);
+
+using Message = std::variant<Request, Response, Notification, Error>;
+
+bool fromJSON(const llvm::json::Value &, Message &, llvm::json::Path);
+llvm::json::Value toJSON(const Message &);
+
+} // namespace lldb_private::mcp::protocol
+
+#endif
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
new file mode 100644
index 000000000000..029d4a887b0c
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.cpp
@@ -0,0 +1,327 @@
+//===- ProtocolServerMCP.cpp ----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "ProtocolServerMCP.h"
+#include "MCPError.h"
+#include "lldb/Core/PluginManager.h"
+#include "lldb/Utility/LLDBLog.h"
+#include "lldb/Utility/Log.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Threading.h"
+#include <thread>
+#include <variant>
+
+using namespace lldb_private;
+using namespace lldb_private::mcp;
+using namespace llvm;
+
+LLDB_PLUGIN_DEFINE(ProtocolServerMCP)
+
+static constexpr size_t kChunkSize = 1024;
+
+ProtocolServerMCP::ProtocolServerMCP(Debugger &debugger)
+    : ProtocolServer(), m_debugger(debugger) {
+  AddRequestHandler("initialize",
+                    std::bind(&ProtocolServerMCP::InitializeHandler, this,
+                              std::placeholders::_1));
+  AddRequestHandler("tools/list",
+                    std::bind(&ProtocolServerMCP::ToolsListHandler, this,
+                              std::placeholders::_1));
+  AddRequestHandler("tools/call",
+                    std::bind(&ProtocolServerMCP::ToolsCallHandler, this,
+                              std::placeholders::_1));
+  AddNotificationHandler(
+      "notifications/initialized", [](const protocol::Notification &) {
+        LLDB_LOG(GetLog(LLDBLog::Host), "MCP initialization complete");
+      });
+  AddTool(std::make_unique<LLDBCommandTool>(
+      "lldb_command", "Run an lldb command.", m_debugger));
+}
+
+ProtocolServerMCP::~ProtocolServerMCP() { llvm::consumeError(Stop()); }
+
+void ProtocolServerMCP::Initialize() {
+  PluginManager::RegisterPlugin(GetPluginNameStatic(),
+                                GetPluginDescriptionStatic(), CreateInstance);
+}
+
+void ProtocolServerMCP::Terminate() {
+  PluginManager::UnregisterPlugin(CreateInstance);
+}
+
+lldb::ProtocolServerSP ProtocolServerMCP::CreateInstance(Debugger &debugger) {
+  return std::make_shared<ProtocolServerMCP>(debugger);
+}
+
+llvm::StringRef ProtocolServerMCP::GetPluginDescriptionStatic() {
+  return "MCP Server.";
+}
+
+llvm::Expected<protocol::Response>
+ProtocolServerMCP::Handle(protocol::Request request) {
+  auto it = m_request_handlers.find(request.method);
+  if (it != m_request_handlers.end()) {
+    llvm::Expected<protocol::Response> response = it->second(request);
+    if (!response)
+      return response;
+    response->id = request.id;
+    return *response;
+  }
+
+  return make_error<MCPError>(
+      llvm::formatv("no handler for request: {0}", request.method).str(), 1);
+}
+
+void ProtocolServerMCP::Handle(protocol::Notification notification) {
+  auto it = m_notification_handlers.find(notification.method);
+  if (it != m_notification_handlers.end()) {
+    it->second(notification);
+    return;
+  }
+
+  LLDB_LOG(GetLog(LLDBLog::Host), "MPC notification: {0} ({1})",
+           notification.method, notification.params);
+}
+
+void ProtocolServerMCP::AcceptCallback(std::unique_ptr<Socket> socket) {
+  LLDB_LOG(GetLog(LLDBLog::Host), "New MCP client ({0}) connected",
+           m_clients.size() + 1);
+
+  lldb::IOObjectSP io_sp = std::move(socket);
+  auto client_up = std::make_unique<Client>();
+  client_up->io_sp = io_sp;
+  Client *client = client_up.get();
+
+  Status status;
+  auto read_handle_up = m_loop.RegisterReadObject(
+      io_sp,
+      [this, client](MainLoopBase &loop) {
+        if (Error error = ReadCallback(*client)) {
+          LLDB_LOG_ERROR(GetLog(LLDBLog::Host), std::move(error), "{0}");
+          client->read_handle_up.reset();
+        }
+      },
+      status);
+  if (status.Fail())
+    return;
+
+  client_up->read_handle_up = std::move(read_handle_up);
+  m_clients.emplace_back(std::move(client_up));
+}
+
+llvm::Error ProtocolServerMCP::ReadCallback(Client &client) {
+  char chunk[kChunkSize];
+  size_t bytes_read = sizeof(chunk);
+  if (Status status = client.io_sp->Read(chunk, bytes_read); status.Fail())
+    return status.takeError();
+  client.buffer.append(chunk, bytes_read);
+
+  for (std::string::size_type pos;
+       (pos = client.buffer.find('\n')) != std::string::npos;) {
+    llvm::Expected<std::optional<protocol::Message>> message =
+        HandleData(StringRef(client.buffer.data(), pos));
+    client.buffer = client.buffer.erase(0, pos + 1);
+    if (!message)
+      return message.takeError();
+
+    if (*message) {
+      std::string Output;
+      llvm::raw_string_ostream OS(Output);
+      OS << llvm::formatv("{0}", toJSON(**message)) << '\n';
+      size_t num_bytes = Output.size();
+      return client.io_sp->Write(Output.data(), num_bytes).takeError();
+    }
+  }
+
+  return llvm::Error::success();
+}
+
+llvm::Error ProtocolServerMCP::Start(ProtocolServer::Connection connection) {
+  std::lock_guard<std::mutex> guard(m_server_mutex);
+
+  if (m_running)
+    return llvm::createStringError("server already running");
+
+  Status status;
+  m_listener = Socket::Create(connection.protocol, status);
+  if (status.Fail())
+    return status.takeError();
+
+  status = m_listener->Listen(connection.name, /*backlog=*/5);
+  if (status.Fail())
+    return status.takeError();
+
+  std::string address =
+      llvm::join(m_listener->GetListeningConnectionURI(), ", ");
+  auto handles =
+      m_listener->Accept(m_loop, std::bind(&ProtocolServerMCP::AcceptCallback,
+                                           this, std::placeholders::_1));
+  if (llvm::Error error = handles.takeError())
+    return error;
+
+  m_listen_handlers = std::move(*handles);
+  m_loop_thread = std::thread([=] {
+    llvm::set_thread_name(
+        llvm::formatv("debugger-{0}.mcp.runloop", m_debugger.GetID()));
+    m_loop.Run();
+  });
+
+  return llvm::Error::success();
+}
+
+llvm::Error ProtocolServerMCP::Stop() {
+  {
+    std::lock_guard<std::mutex> guard(m_server_mutex);
+    m_running = false;
+  }
+
+  // Stop the main loop.
+  m_loop.AddPendingCallback(
+      [](MainLoopBase &loop) { loop.RequestTermination(); });
+
+  // Wait for the main loop to exit.
+  if (m_loop_thread.joinable())
+    m_loop_thread.join();
+
+  {
+    std::lock_guard<std::mutex> guard(m_server_mutex);
+    m_listener.reset();
+    m_listen_handlers.clear();
+    m_clients.clear();
+  }
+
+  return llvm::Error::success();
+}
+
+llvm::Expected<std::optional<protocol::Message>>
+ProtocolServerMCP::HandleData(llvm::StringRef data) {
+  auto message = llvm::json::parse<protocol::Message>(/*JSON=*/data);
+  if (!message)
+    return message.takeError();
+
+  if (const protocol::Request *request =
+          std::get_if<protocol::Request>(&(*message))) {
+    llvm::Expected<protocol::Response> response = Handle(*request);
+
+    // Handle failures by converting them into an Error message.
+    if (!response) {
+      protocol::Error protocol_error;
+      llvm::handleAllErrors(
+          response.takeError(),
+          [&](const MCPError &err) { protocol_error = err.toProtcolError(); },
+          [&](const llvm::ErrorInfoBase &err) {
+            protocol_error.error.code = -1;
+            protocol_error.error.message = err.message();
+          });
+      protocol_error.id = request->id;
+      return protocol_error;
+    }
+
+    return *response;
+  }
+
+  if (const protocol::Notification *notification =
+          std::get_if<protocol::Notification>(&(*message))) {
+    Handle(*notification);
+    return std::nullopt;
+  }
+
+  if (std::get_if<protocol::Error>(&(*message)))
+    return llvm::createStringError("unexpected MCP message: error");
+
+  if (std::get_if<protocol::Response>(&(*message)))
+    return llvm::createStringError("unexpected MCP message: response");
+
+  llvm_unreachable("all message types handled");
+}
+
+protocol::Capabilities ProtocolServerMCP::GetCapabilities() {
+  protocol::Capabilities capabilities;
+  capabilities.tools.listChanged = true;
+  return capabilities;
+}
+
+void ProtocolServerMCP::AddTool(std::unique_ptr<Tool> tool) {
+  std::lock_guard<std::mutex> guard(m_server_mutex);
+
+  if (!tool)
+    return;
+  m_tools[tool->GetName()] = std::move(tool);
+}
+
+void ProtocolServerMCP::AddRequestHandler(llvm::StringRef method,
+                                          RequestHandler handler) {
+  std::lock_guard<std::mutex> guard(m_server_mutex);
+  m_request_handlers[method] = std::move(handler);
+}
+
+void ProtocolServerMCP::AddNotificationHandler(llvm::StringRef method,
+                                               NotificationHandler handler) {
+  std::lock_guard<std::mutex> guard(m_server_mutex);
+  m_notification_handlers[method] = std::move(handler);
+}
+
+llvm::Expected<protocol::Response>
+ProtocolServerMCP::InitializeHandler(const protocol::Request &request) {
+  protocol::Response response;
+  response.result.emplace(llvm::json::Object{
+      {"protocolVersion", protocol::kVersion},
+      {"capabilities", GetCapabilities()},
+      {"serverInfo",
+       llvm::json::Object{{"name", kName}, {"version", kVersion}}}});
+  return response;
+}
+
+llvm::Expected<protocol::Response>
+ProtocolServerMCP::ToolsListHandler(const protocol::Request &request) {
+  protocol::Response response;
+
+  llvm::json::Array tools;
+  for (const auto &tool : m_tools)
+    tools.emplace_back(toJSON(tool.second->GetDefinition()));
+
+  response.result.emplace(llvm::json::Object{{"tools", std::move(tools)}});
+
+  return response;
+}
+
+llvm::Expected<protocol::Response>
+ProtocolServerMCP::ToolsCallHandler(const protocol::Request &request) {
+  protocol::Response response;
+
+  if (!request.params)
+    return llvm::createStringError("no tool parameters");
+
+  const json::Object *param_obj = request.params->getAsObject();
+  if (!param_obj)
+    return llvm::createStringError("no tool parameters");
+
+  const json::Value *name = param_obj->get("name");
+  if (!name)
+    return llvm::createStringError("no tool name");
+
+  llvm::StringRef tool_name = name->getAsString().value_or("");
+  if (tool_name.empty())
+    return llvm::createStringError("no tool name");
+
+  auto it = m_tools.find(tool_name);
+  if (it == m_tools.end())
+    return llvm::createStringError(llvm::formatv("no tool \"{0}\"", tool_name));
+
+  const json::Value *args = param_obj->get("arguments");
+  if (!args)
+    return llvm::createStringError("no tool arguments");
+
+  llvm::Expected<protocol::TextResult> text_result = it->second->Call(*args);
+  if (!text_result)
+    return text_result.takeError();
+
+  response.result.emplace(toJSON(*text_result));
+
+  return response;
+}
diff --git a/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
new file mode 100644
index 000000000000..52bb92a04a80
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/ProtocolServerMCP.h
@@ -0,0 +1,100 @@
+//===- ProtocolServerMCP.h ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_PROTOCOL_MCP_PROTOCOLSERVERMCP_H
+#define LLDB_PLUGINS_PROTOCOL_MCP_PROTOCOLSERVERMCP_H
+
+#include "Protocol.h"
+#include "Tool.h"
+#include "lldb/Core/ProtocolServer.h"
+#include "lldb/Host/MainLoop.h"
+#include "lldb/Host/Socket.h"
+#include "llvm/ADT/StringMap.h"
+#include <thread>
+
+namespace lldb_private::mcp {
+
+class ProtocolServerMCP : public ProtocolServer {
+public:
+  ProtocolServerMCP(Debugger &debugger);
+  virtual ~ProtocolServerMCP() override;
+
+  virtual llvm::Error Start(ProtocolServer::Connection connection) override;
+  virtual llvm::Error Stop() override;
+
+  static void Initialize();
+  static void Terminate();
+
+  static llvm::StringRef GetPluginNameStatic() { return "MCP"; }
+  static llvm::StringRef GetPluginDescriptionStatic();
+
+  static lldb::ProtocolServerSP CreateInstance(Debugger &debugger);
+
+  llvm::StringRef GetPluginName() override { return GetPluginNameStatic(); }
+
+  Socket *GetSocket() const override { return m_listener.get(); }
+
+protected:
+  using RequestHandler = std::function<llvm::Expected<protocol::Response>(
+      const protocol::Request &)>;
+  using NotificationHandler =
+      std::function<void(const protocol::Notification &)>;
+
+  void AddTool(std::unique_ptr<Tool> tool);
+  void AddRequestHandler(llvm::StringRef method, RequestHandler handler);
+  void AddNotificationHandler(llvm::StringRef method,
+                              NotificationHandler handler);
+
+private:
+  void AcceptCallback(std::unique_ptr<Socket> socket);
+
+  llvm::Expected<std::optional<protocol::Message>>
+  HandleData(llvm::StringRef data);
+
+  llvm::Expected<protocol::Response> Handle(protocol::Request request);
+  void Handle(protocol::Notification notification);
+
+  llvm::Expected<protocol::Response>
+  InitializeHandler(const protocol::Request &);
+  llvm::Expected<protocol::Response>
+  ToolsListHandler(const protocol::Request &);
+  llvm::Expected<protocol::Response>
+  ToolsCallHandler(const protocol::Request &);
+
+  protocol::Capabilities GetCapabilities();
+
+  llvm::StringLiteral kName = "lldb-mcp";
+  llvm::StringLiteral kVersion = "0.1.0";
+
+  Debugger &m_debugger;
+
+  bool m_running = false;
+
+  MainLoop m_loop;
+  std::thread m_loop_thread;
+
+  std::unique_ptr<Socket> m_listener;
+  std::vector<MainLoopBase::ReadHandleUP> m_listen_handlers;
+
+  struct Client {
+    lldb::IOObjectSP io_sp;
+    MainLoopBase::ReadHandleUP read_handle_up;
+    std::string buffer;
+  };
+  llvm::Error ReadCallback(Client &client);
+  std::vector<std::unique_ptr<Client>> m_clients;
+
+  std::mutex m_server_mutex;
+  llvm::StringMap<std::unique_ptr<Tool>> m_tools;
+
+  llvm::StringMap<RequestHandler> m_request_handlers;
+  llvm::StringMap<NotificationHandler> m_notification_handlers;
+};
+} // namespace lldb_private::mcp
+
+#endif
diff --git a/lldb/source/Plugins/Protocol/MCP/Tool.cpp b/lldb/source/Plugins/Protocol/MCP/Tool.cpp
new file mode 100644
index 000000000000..de8fcc8f3cb4
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/Tool.cpp
@@ -0,0 +1,81 @@
+//===- Tool.cpp -----------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Tool.h"
+#include "lldb/Interpreter/CommandInterpreter.h"
+#include "lldb/Interpreter/CommandReturnObject.h"
+
+using namespace lldb_private::mcp;
+using namespace llvm;
+
+struct LLDBCommandToolArguments {
+  std::string arguments;
+};
+
+bool fromJSON(const llvm::json::Value &V, LLDBCommandToolArguments &A,
+              llvm::json::Path P) {
+  llvm::json::ObjectMapper O(V, P);
+  return O && O.map("arguments", A.arguments);
+}
+
+Tool::Tool(std::string name, std::string description)
+    : m_name(std::move(name)), m_description(std::move(description)) {}
+
+protocol::ToolDefinition Tool::GetDefinition() const {
+  protocol::ToolDefinition definition;
+  definition.name = m_name;
+  definition.description.emplace(m_description);
+
+  if (std::optional<llvm::json::Value> input_schema = GetSchema())
+    definition.inputSchema = *input_schema;
+
+  return definition;
+}
+
+LLDBCommandTool::LLDBCommandTool(std::string name, std::string description,
+                                 Debugger &debugger)
+    : Tool(std::move(name), std::move(description)), m_debugger(debugger) {}
+
+llvm::Expected<protocol::TextResult>
+LLDBCommandTool::Call(const llvm::json::Value &args) {
+  llvm::json::Path::Root root;
+
+  LLDBCommandToolArguments arguments;
+  if (!fromJSON(args, arguments, root))
+    return root.getError();
+
+  // FIXME: Disallow certain commands and their aliases.
+  CommandReturnObject result(/*colors=*/false);
+  m_debugger.GetCommandInterpreter().HandleCommand(arguments.arguments.c_str(),
+                                                   eLazyBoolYes, result);
+
+  std::string output;
+  llvm::StringRef output_str = result.GetOutputString();
+  if (!output_str.empty())
+    output += output_str.str();
+
+  std::string err_str = result.GetErrorString();
+  if (!err_str.empty()) {
+    if (!output.empty())
+      output += '\n';
+    output += err_str;
+  }
+
+  mcp::protocol::TextResult text_result;
+  text_result.content.emplace_back(mcp::protocol::TextContent{{output}});
+  text_result.isError = !result.Succeeded();
+  return text_result;
+}
+
+std::optional<llvm::json::Value> LLDBCommandTool::GetSchema() const {
+  llvm::json::Object str_type{{"type", "string"}};
+  llvm::json::Object properties{{"arguments", std::move(str_type)}};
+  llvm::json::Object schema{{"type", "object"},
+                            {"properties", std::move(properties)}};
+  return schema;
+}
diff --git a/lldb/source/Plugins/Protocol/MCP/Tool.h b/lldb/source/Plugins/Protocol/MCP/Tool.h
new file mode 100644
index 000000000000..57a5125813b7
--- /dev/null
+++ b/lldb/source/Plugins/Protocol/MCP/Tool.h
@@ -0,0 +1,56 @@
+//===- Tool.h -------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLDB_PLUGINS_PROTOCOL_MCP_TOOL_H
+#define LLDB_PLUGINS_PROTOCOL_MCP_TOOL_H
+
+#include "Protocol.h"
+#include "lldb/Core/Debugger.h"
+#include "llvm/Support/JSON.h"
+#include <string>
+
+namespace lldb_private::mcp {
+
+class Tool {
+public:
+  Tool(std::string name, std::string description);
+  virtual ~Tool() = default;
+
+  virtual llvm::Expected<protocol::TextResult>
+  Call(const llvm::json::Value &args) = 0;
+
+  virtual std::optional<llvm::json::Value> GetSchema() const {
+    return std::nullopt;
+  }
+
+  protocol::ToolDefinition GetDefinition() const;
+
+  const std::string &GetName() { return m_name; }
+
+private:
+  std::string m_name;
+  std::string m_description;
+};
+
+class LLDBCommandTool : public mcp::Tool {
+public:
+  LLDBCommandTool(std::string name, std::string description,
+                  Debugger &debugger);
+  ~LLDBCommandTool() = default;
+
+  virtual llvm::Expected<protocol::TextResult>
+  Call(const llvm::json::Value &args) override;
+
+  virtual std::optional<llvm::json::Value> GetSchema() const override;
+
+private:
+  Debugger &m_debugger;
+};
+} // namespace lldb_private::mcp
+
+#endif
diff --git a/lldb/unittests/CMakeLists.txt b/lldb/unittests/CMakeLists.txt
index 6eaaa4f4c8c9..b48b9bafe3bc 100644
--- a/lldb/unittests/CMakeLists.txt
+++ b/lldb/unittests/CMakeLists.txt
@@ -78,6 +78,10 @@ add_subdirectory(Utility)
 add_subdirectory(Thread)
 add_subdirectory(ValueObject)
 
+if(LLDB_ENABLE_PROTOCOL_SERVERS)
+  add_subdirectory(Protocol)
+endif()
+
 if(LLDB_CAN_USE_DEBUGSERVER AND LLDB_TOOL_DEBUGSERVER_BUILD AND NOT LLDB_USE_SYSTEM_DEBUGSERVER)
   add_subdirectory(debugserver)
 endif()
diff --git a/lldb/unittests/DAP/ProtocolTypesTest.cpp b/lldb/unittests/DAP/ProtocolTypesTest.cpp
index 9c93eb8c94b0..085348ffc519 100644
--- a/lldb/unittests/DAP/ProtocolTypesTest.cpp
+++ b/lldb/unittests/DAP/ProtocolTypesTest.cpp
@@ -9,6 +9,7 @@
 #include "Protocol/ProtocolTypes.h"
 #include "Protocol/ProtocolEvents.h"
 #include "Protocol/ProtocolRequests.h"
+#include "TestingSupport/TestUtilities.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/JSON.h"
 #include "llvm/Testing/Support/Error.h"
@@ -20,6 +21,7 @@ using namespace llvm;
 using namespace lldb;
 using namespace lldb_dap;
 using namespace lldb_dap::protocol;
+using lldb_private::roundtripJSON;
 using llvm::json::parse;
 using llvm::json::Value;
 
@@ -28,15 +30,6 @@ static std::string pp(const json::Value &E) {
   return formatv("{0:2}", E).str();
 }
 
-template <typename T> static llvm::Expected<T> roundtrip(const T &input) {
-  llvm::json::Value value = toJSON(input);
-  llvm::json::Path::Root root;
-  T output;
-  if (!fromJSON(value, output, root))
-    return root.getError();
-  return output;
-}
-
 TEST(ProtocolTypesTest, ExceptionBreakpointsFilter) {
   ExceptionBreakpointsFilter filter;
   filter.filter = "testFilter";
@@ -47,7 +40,7 @@ TEST(ProtocolTypesTest, ExceptionBreakpointsFilter) {
   filter.conditionDescription = "Condition for test filter";
 
   llvm::Expected<ExceptionBreakpointsFilter> deserialized_filter =
-      roundtrip(filter);
+      roundtripJSON(filter);
   ASSERT_THAT_EXPECTED(deserialized_filter, llvm::Succeeded());
 
   EXPECT_EQ(filter.filter, deserialized_filter->filter);
@@ -66,7 +59,7 @@ TEST(ProtocolTypesTest, Source) {
   source.sourceReference = 12345;
   source.presentationHint = Source::eSourcePresentationHintEmphasize;
 
-  llvm::Expected<Source> deserialized_source = roundtrip(source);
+  llvm::Expected<Source> deserialized_source = roundtripJSON(source);
   ASSERT_THAT_EXPECTED(deserialized_source, llvm::Succeeded());
 
   EXPECT_EQ(source.name, deserialized_source->name);
@@ -83,7 +76,7 @@ TEST(ProtocolTypesTest, ColumnDescriptor) {
   column.type = eColumnTypeString;
   column.width = 20;
 
-  llvm::Expected<ColumnDescriptor> deserialized_column = roundtrip(column);
+  llvm::Expected<ColumnDescriptor> deserialized_column = roundtripJSON(column);
   ASSERT_THAT_EXPECTED(deserialized_column, llvm::Succeeded());
 
   EXPECT_EQ(column.attributeName, deserialized_column->attributeName);
@@ -101,7 +94,7 @@ TEST(ProtocolTypesTest, BreakpointMode) {
   mode.appliesTo = {eBreakpointModeApplicabilitySource,
                     eBreakpointModeApplicabilityException};
 
-  llvm::Expected<BreakpointMode> deserialized_mode = roundtrip(mode);
+  llvm::Expected<BreakpointMode> deserialized_mode = roundtripJSON(mode);
   ASSERT_THAT_EXPECTED(deserialized_mode, llvm::Succeeded());
 
   EXPECT_EQ(mode.mode, deserialized_mode->mode);
@@ -125,7 +118,8 @@ TEST(ProtocolTypesTest, Breakpoint) {
   breakpoint.offset = 4;
   breakpoint.reason = BreakpointReason::eBreakpointReasonPending;
 
-  llvm::Expected<Breakpoint> deserialized_breakpoint = roundtrip(breakpoint);
+  llvm::Expected<Breakpoint> deserialized_breakpoint =
+      roundtripJSON(breakpoint);
   ASSERT_THAT_EXPECTED(deserialized_breakpoint, llvm::Succeeded());
 
   EXPECT_EQ(breakpoint.id, deserialized_breakpoint->id);
@@ -157,7 +151,7 @@ TEST(ProtocolTypesTest, SourceBreakpoint) {
   source_breakpoint.mode = "hardware";
 
   llvm::Expected<SourceBreakpoint> deserialized_source_breakpoint =
-      roundtrip(source_breakpoint);
+      roundtripJSON(source_breakpoint);
   ASSERT_THAT_EXPECTED(deserialized_source_breakpoint, llvm::Succeeded());
 
   EXPECT_EQ(source_breakpoint.line, deserialized_source_breakpoint->line);
@@ -178,7 +172,7 @@ TEST(ProtocolTypesTest, FunctionBreakpoint) {
   function_breakpoint.hitCondition = "3";
 
   llvm::Expected<FunctionBreakpoint> deserialized_function_breakpoint =
-      roundtrip(function_breakpoint);
+      roundtripJSON(function_breakpoint);
   ASSERT_THAT_EXPECTED(deserialized_function_breakpoint, llvm::Succeeded());
 
   EXPECT_EQ(function_breakpoint.name, deserialized_function_breakpoint->name);
@@ -196,7 +190,7 @@ TEST(ProtocolTypesTest, DataBreakpoint) {
   data_breakpoint_info.hitCondition = "10";
 
   llvm::Expected<DataBreakpoint> deserialized_data_breakpoint_info =
-      roundtrip(data_breakpoint_info);
+      roundtripJSON(data_breakpoint_info);
   ASSERT_THAT_EXPECTED(deserialized_data_breakpoint_info, llvm::Succeeded());
 
   EXPECT_EQ(data_breakpoint_info.dataId,
@@ -233,9 +227,9 @@ TEST(ProtocolTypesTest, Capabilities) {
                                    {eBreakpointModeApplicabilitySource}}};
   capabilities.lldbExtVersion = "1.0.0";
 
-  // Perform roundtrip serialization and deserialization.
+  // Perform roundtripJSON serialization and deserialization.
   llvm::Expected<Capabilities> deserialized_capabilities =
-      roundtrip(capabilities);
+      roundtripJSON(capabilities);
   ASSERT_THAT_EXPECTED(deserialized_capabilities, llvm::Succeeded());
 
   // Verify supported features.
@@ -316,7 +310,7 @@ TEST(ProtocolTypesTest, Scope) {
   source.presentationHint = Source::eSourcePresentationHintNormal;
   scope.source = source;
 
-  llvm::Expected<Scope> deserialized_scope = roundtrip(scope);
+  llvm::Expected<Scope> deserialized_scope = roundtripJSON(scope);
   ASSERT_THAT_EXPECTED(deserialized_scope, llvm::Succeeded());
   EXPECT_EQ(scope.name, deserialized_scope->name);
   EXPECT_EQ(scope.presentationHint, deserialized_scope->presentationHint);
@@ -755,7 +749,7 @@ TEST(ProtocolTypesTest, StepInTarget) {
   target.endLine = 32;
   target.endColumn = 23;
 
-  llvm::Expected<StepInTarget> deserialized_target = roundtrip(target);
+  llvm::Expected<StepInTarget> deserialized_target = roundtripJSON(target);
   ASSERT_THAT_EXPECTED(deserialized_target, llvm::Succeeded());
 
   EXPECT_EQ(target.id, deserialized_target->id);
@@ -797,4 +791,4 @@ TEST(ProtocolTypesTest, ReadMemoryResponseBody) {
       R"({ "address": "0xdeadbeef", "data": "aGVsbG8gd29ybGQh", "unreadableBytes": 1})");
   ASSERT_THAT_EXPECTED(expected, llvm::Succeeded());
   EXPECT_EQ(pp(*expected), pp(response));
-}
\ No newline at end of file
+}
diff --git a/lldb/unittests/Protocol/CMakeLists.txt b/lldb/unittests/Protocol/CMakeLists.txt
new file mode 100644
index 000000000000..801662b0544d
--- /dev/null
+++ b/lldb/unittests/Protocol/CMakeLists.txt
@@ -0,0 +1,12 @@
+add_lldb_unittest(ProtocolTests
+  ProtocolMCPTest.cpp
+  ProtocolMCPServerTest.cpp
+
+  LINK_LIBS
+    lldbCore
+    lldbUtility
+    lldbHost
+    lldbPluginPlatformMacOSX
+    lldbPluginProtocolServerMCP
+    LLVMTestingSupport
+  )
diff --git a/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp b/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
new file mode 100644
index 000000000000..72b8c7b1fd82
--- /dev/null
+++ b/lldb/unittests/Protocol/ProtocolMCPServerTest.cpp
@@ -0,0 +1,291 @@
+//===-- ProtocolServerMCPTest.cpp -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/Platform/MacOSX/PlatformRemoteMacOSX.h"
+#include "Plugins/Protocol/MCP/ProtocolServerMCP.h"
+#include "TestingSupport/Host/SocketTestUtilities.h"
+#include "TestingSupport/SubsystemRAII.h"
+#include "lldb/Core/ProtocolServer.h"
+#include "lldb/Host/FileSystem.h"
+#include "lldb/Host/HostInfo.h"
+#include "lldb/Host/JSONTransport.h"
+#include "lldb/Host/Socket.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::mcp::protocol;
+
+namespace {
+class TestProtocolServerMCP : public lldb_private::mcp::ProtocolServerMCP {
+public:
+  using ProtocolServerMCP::AddNotificationHandler;
+  using ProtocolServerMCP::AddRequestHandler;
+  using ProtocolServerMCP::AddTool;
+  using ProtocolServerMCP::GetSocket;
+  using ProtocolServerMCP::ProtocolServerMCP;
+};
+
+class TestJSONTransport : public lldb_private::JSONRPCTransport {
+public:
+  using JSONRPCTransport::JSONRPCTransport;
+  using JSONRPCTransport::ReadImpl;
+  using JSONRPCTransport::WriteImpl;
+};
+
+/// Test tool that returns it argument as text.
+class TestTool : public mcp::Tool {
+public:
+  using mcp::Tool::Tool;
+
+  virtual llvm::Expected<mcp::protocol::TextResult>
+  Call(const llvm::json::Value &args) override {
+    std::string argument;
+    if (const json::Object *args_obj = args.getAsObject()) {
+      if (const json::Value *s = args_obj->get("arguments")) {
+        argument = s->getAsString().value_or("");
+      }
+    }
+
+    mcp::protocol::TextResult text_result;
+    text_result.content.emplace_back(mcp::protocol::TextContent{{argument}});
+    return text_result;
+  }
+};
+
+/// Test tool that returns an error.
+class ErrorTool : public mcp::Tool {
+public:
+  using mcp::Tool::Tool;
+
+  virtual llvm::Expected<mcp::protocol::TextResult>
+  Call(const llvm::json::Value &args) override {
+    return llvm::createStringError("error");
+  }
+};
+
+/// Test tool that fails but doesn't return an error.
+class FailTool : public mcp::Tool {
+public:
+  using mcp::Tool::Tool;
+
+  virtual llvm::Expected<mcp::protocol::TextResult>
+  Call(const llvm::json::Value &args) override {
+    mcp::protocol::TextResult text_result;
+    text_result.content.emplace_back(mcp::protocol::TextContent{{"failed"}});
+    text_result.isError = true;
+    return text_result;
+  }
+};
+
+class ProtocolServerMCPTest : public ::testing::Test {
+public:
+  SubsystemRAII<FileSystem, HostInfo, PlatformRemoteMacOSX, Socket> subsystems;
+  DebuggerSP m_debugger_sp;
+
+  lldb::IOObjectSP m_io_sp;
+  std::unique_ptr<TestJSONTransport> m_transport_up;
+  std::unique_ptr<TestProtocolServerMCP> m_server_up;
+
+  static constexpr llvm::StringLiteral k_localhost = "localhost";
+
+  llvm::Error Write(llvm::StringRef message) {
+    return m_transport_up->WriteImpl(llvm::formatv("{0}\n", message).str());
+  }
+
+  llvm::Expected<std::string> Read() {
+    return m_transport_up->ReadImpl(std::chrono::milliseconds(100));
+  }
+
+  void SetUp() {
+    // Create a debugger.
+    ArchSpec arch("arm64-apple-macosx-");
+    Platform::SetHostPlatform(
+        PlatformRemoteMacOSX::CreateInstance(true, &arch));
+    m_debugger_sp = Debugger::CreateInstance();
+
+    // Create & start the server.
+    ProtocolServer::Connection connection;
+    connection.protocol = Socket::SocketProtocol::ProtocolTcp;
+    connection.name = llvm::formatv("{0}:0", k_localhost).str();
+    m_server_up = std::make_unique<TestProtocolServerMCP>(*m_debugger_sp);
+    m_server_up->AddTool(std::make_unique<TestTool>("test", "test tool"));
+    ASSERT_THAT_ERROR(m_server_up->Start(connection), llvm::Succeeded());
+
+    // Connect to the server over a TCP socket.
+    auto connect_socket_up = std::make_unique<TCPSocket>(true);
+    ASSERT_THAT_ERROR(connect_socket_up
+                          ->Connect(llvm::formatv("{0}:{1}", k_localhost,
+                                                  static_cast<TCPSocket *>(
+                                                      m_server_up->GetSocket())
+                                                      ->GetLocalPortNumber())
+                                        .str())
+                          .ToError(),
+                      llvm::Succeeded());
+
+    // Set up JSON transport for the client.
+    m_io_sp = std::move(connect_socket_up);
+    m_transport_up = std::make_unique<TestJSONTransport>(m_io_sp, m_io_sp);
+  }
+
+  void TearDown() {
+    // Stop the server.
+    ASSERT_THAT_ERROR(m_server_up->Stop(), llvm::Succeeded());
+  }
+};
+
+} // namespace
+
+TEST_F(ProtocolServerMCPTest, Intialization) {
+  llvm::StringLiteral request =
+      R"json({"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"claude-ai","version":"0.1.0"}},"jsonrpc":"2.0","id":0})json";
+  llvm::StringLiteral response =
+      R"json({"jsonrpc":"2.0","id":0,"result":{"capabilities":{"tools":{"listChanged":true}},"protocolVersion":"2024-11-05","serverInfo":{"name":"lldb-mcp","version":"0.1.0"}}})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, ToolsList) {
+  llvm::StringLiteral request =
+      R"json({"method":"tools/list","params":{},"jsonrpc":"2.0","id":1})json";
+  llvm::StringLiteral response =
+      R"json({"id":1,"jsonrpc":"2.0","result":{"tools":[{"description":"test tool","name":"test"},{"description":"Run an lldb command.","inputSchema":{"properties":{"arguments":{"type":"string"}},"type":"object"},"name":"lldb_command"}]}})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, ResourcesList) {
+  llvm::StringLiteral request =
+      R"json({"method":"resources/list","params":{},"jsonrpc":"2.0","id":2})json";
+  llvm::StringLiteral response =
+      R"json({"error":{"code":1,"message":"no handler for request: resources/list"},"id":2,"jsonrpc":"2.0"})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, ToolsCall) {
+  llvm::StringLiteral request =
+      R"json({"method":"tools/call","params":{"name":"test","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+  llvm::StringLiteral response =
+      R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"foo","type":"text"}],"isError":false}})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, ToolsCallError) {
+  m_server_up->AddTool(std::make_unique<ErrorTool>("error", "error tool"));
+
+  llvm::StringLiteral request =
+      R"json({"method":"tools/call","params":{"name":"error","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+  llvm::StringLiteral response =
+      R"json({"error":{"code":-1,"message":"error"},"id":11,"jsonrpc":"2.0"})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, ToolsCallFail) {
+  m_server_up->AddTool(std::make_unique<FailTool>("fail", "fail tool"));
+
+  llvm::StringLiteral request =
+      R"json({"method":"tools/call","params":{"name":"fail","arguments":{"arguments":"foo"}},"jsonrpc":"2.0","id":11})json";
+  llvm::StringLiteral response =
+      R"json({"id":11,"jsonrpc":"2.0","result":{"content":[{"text":"failed","type":"text"}],"isError":true}})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  llvm::Expected<std::string> response_str = Read();
+  ASSERT_THAT_EXPECTED(response_str, llvm::Succeeded());
+
+  llvm::Expected<json::Value> response_json = json::parse(*response_str);
+  ASSERT_THAT_EXPECTED(response_json, llvm::Succeeded());
+
+  llvm::Expected<json::Value> expected_json = json::parse(response);
+  ASSERT_THAT_EXPECTED(expected_json, llvm::Succeeded());
+
+  EXPECT_EQ(*response_json, *expected_json);
+}
+
+TEST_F(ProtocolServerMCPTest, NotificationInitialized) {
+  bool handler_called = false;
+  std::condition_variable cv;
+  std::mutex mutex;
+
+  m_server_up->AddNotificationHandler(
+      "notifications/initialized",
+      [&](const mcp::protocol::Notification &notification) {
+        {
+          std::lock_guard<std::mutex> lock(mutex);
+          handler_called = true;
+        }
+        cv.notify_all();
+      });
+  llvm::StringLiteral request =
+      R"json({"method":"notifications/initialized","jsonrpc":"2.0"})json";
+
+  ASSERT_THAT_ERROR(Write(request), llvm::Succeeded());
+
+  std::unique_lock<std::mutex> lock(mutex);
+  cv.wait(lock, [&] { return handler_called; });
+}
diff --git a/lldb/unittests/Protocol/ProtocolMCPTest.cpp b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
new file mode 100644
index 000000000000..00959f3ce20b
--- /dev/null
+++ b/lldb/unittests/Protocol/ProtocolMCPTest.cpp
@@ -0,0 +1,135 @@
+//===-- ProtocolMCPTest.cpp -----------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "Plugins/Protocol/MCP/Protocol.h"
+#include "TestingSupport/TestUtilities.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+
+using namespace lldb;
+using namespace lldb_private;
+using namespace lldb_private::mcp::protocol;
+
+TEST(ProtocolMCPTest, Request) {
+  Request request;
+  request.id = 1;
+  request.method = "foo";
+  request.params = llvm::json::Object{{"key", "value"}};
+
+  llvm::Expected<Request> deserialized_request = roundtripJSON(request);
+  ASSERT_THAT_EXPECTED(deserialized_request, llvm::Succeeded());
+
+  EXPECT_EQ(request.id, deserialized_request->id);
+  EXPECT_EQ(request.method, deserialized_request->method);
+  EXPECT_EQ(request.params, deserialized_request->params);
+}
+
+TEST(ProtocolMCPTest, Response) {
+  Response response;
+  response.id = 1;
+  response.result = llvm::json::Object{{"key", "value"}};
+
+  llvm::Expected<Response> deserialized_response = roundtripJSON(response);
+  ASSERT_THAT_EXPECTED(deserialized_response, llvm::Succeeded());
+
+  EXPECT_EQ(response.id, deserialized_response->id);
+  EXPECT_EQ(response.result, deserialized_response->result);
+}
+
+TEST(ProtocolMCPTest, Notification) {
+  Notification notification;
+  notification.method = "notifyMethod";
+  notification.params = llvm::json::Object{{"key", "value"}};
+
+  llvm::Expected<Notification> deserialized_notification =
+      roundtripJSON(notification);
+  ASSERT_THAT_EXPECTED(deserialized_notification, llvm::Succeeded());
+
+  EXPECT_EQ(notification.method, deserialized_notification->method);
+  EXPECT_EQ(notification.params, deserialized_notification->params);
+}
+
+TEST(ProtocolMCPTest, ToolCapability) {
+  ToolCapability tool_capability;
+  tool_capability.listChanged = true;
+
+  llvm::Expected<ToolCapability> deserialized_tool_capability =
+      roundtripJSON(tool_capability);
+  ASSERT_THAT_EXPECTED(deserialized_tool_capability, llvm::Succeeded());
+
+  EXPECT_EQ(tool_capability.listChanged,
+            deserialized_tool_capability->listChanged);
+}
+
+TEST(ProtocolMCPTest, Capabilities) {
+  ToolCapability tool_capability;
+  tool_capability.listChanged = true;
+
+  Capabilities capabilities;
+  capabilities.tools = tool_capability;
+
+  llvm::Expected<Capabilities> deserialized_capabilities =
+      roundtripJSON(capabilities);
+  ASSERT_THAT_EXPECTED(deserialized_capabilities, llvm::Succeeded());
+
+  EXPECT_EQ(capabilities.tools.listChanged,
+            deserialized_capabilities->tools.listChanged);
+}
+
+TEST(ProtocolMCPTest, TextContent) {
+  TextContent text_content;
+  text_content.text = "Sample text";
+
+  llvm::Expected<TextContent> deserialized_text_content =
+      roundtripJSON(text_content);
+  ASSERT_THAT_EXPECTED(deserialized_text_content, llvm::Succeeded());
+
+  EXPECT_EQ(text_content.text, deserialized_text_content->text);
+}
+
+TEST(ProtocolMCPTest, TextResult) {
+  TextContent text_content1;
+  text_content1.text = "Text 1";
+
+  TextContent text_content2;
+  text_content2.text = "Text 2";
+
+  TextResult text_result;
+  text_result.content = {text_content1, text_content2};
+  text_result.isError = true;
+
+  llvm::Expected<TextResult> deserialized_text_result =
+      roundtripJSON(text_result);
+  ASSERT_THAT_EXPECTED(deserialized_text_result, llvm::Succeeded());
+
+  EXPECT_EQ(text_result.isError, deserialized_text_result->isError);
+  ASSERT_EQ(text_result.content.size(),
+            deserialized_text_result->content.size());
+  EXPECT_EQ(text_result.content[0].text,
+            deserialized_text_result->content[0].text);
+  EXPECT_EQ(text_result.content[1].text,
+            deserialized_text_result->content[1].text);
+}
+
+TEST(ProtocolMCPTest, ToolDefinition) {
+  ToolDefinition tool_definition;
+  tool_definition.name = "ToolName";
+  tool_definition.description = "Tool Description";
+  tool_definition.inputSchema =
+      llvm::json::Object{{"schemaKey", "schemaValue"}};
+
+  llvm::Expected<ToolDefinition> deserialized_tool_definition =
+      roundtripJSON(tool_definition);
+  ASSERT_THAT_EXPECTED(deserialized_tool_definition, llvm::Succeeded());
+
+  EXPECT_EQ(tool_definition.name, deserialized_tool_definition->name);
+  EXPECT_EQ(tool_definition.description,
+            deserialized_tool_definition->description);
+  EXPECT_EQ(tool_definition.inputSchema,
+            deserialized_tool_definition->inputSchema);
+}
diff --git a/lldb/unittests/TestingSupport/TestUtilities.h b/lldb/unittests/TestingSupport/TestUtilities.h
index 65994384059f..db62881872fe 100644
--- a/lldb/unittests/TestingSupport/TestUtilities.h
+++ b/lldb/unittests/TestingSupport/TestUtilities.h
@@ -59,6 +59,15 @@ private:
 
   std::string Buffer;
 };
+
+template <typename T> static llvm::Expected<T> roundtripJSON(const T &input) {
+  llvm::json::Value value = toJSON(input);
+  llvm::json::Path::Root root;
+  T output;
+  if (!fromJSON(value, output, root))
+    return root.getError();
+  return output;
+}
 } // namespace lldb_private
 
 #endif
-- 
cgit v1.2.3


From 78971916da04895838ff043f4fc71760dcca5bac Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Fri, 20 Jun 2025 15:48:30 +0000
Subject: [gn build] Port 9524bfb27020

---
 llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn | 1 +
 llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn     | 1 +
 2 files changed, 2 insertions(+)

diff --git a/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
index 1fc119c07244..f92438977045 100644
--- a/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Commands/BUILD.gn
@@ -50,6 +50,7 @@ static_library("Commands") {
     "CommandObjectPlatform.cpp",
     "CommandObjectPlugin.cpp",
     "CommandObjectProcess.cpp",
+    "CommandObjectProtocolServer.cpp",
     "CommandObjectQuit.cpp",
     "CommandObjectRegexCommand.cpp",
     "CommandObjectRegister.cpp",
diff --git a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
index 36d6f270d7e5..5ae699a33ea4 100644
--- a/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/lldb/source/Core/BUILD.gn
@@ -74,6 +74,7 @@ static_library("Core") {
     "Opcode.cpp",
     "PluginManager.cpp",
     "Progress.cpp",
+    "ProtocolServer.cpp",
     "RichManglingContext.cpp",
     "SearchFilter.cpp",
     "Section.cpp",
-- 
cgit v1.2.3


From 6e86b7e34b9494a01bf7164825c3d72ff21a4c7f Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Fri, 20 Jun 2025 16:52:43 +0100
Subject: [AMDGPU] Do not replace SALU floating point multiply with VALU-only
 ldexp (#145048)

---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp          |  6 ++
 .../CodeGen/AMDGPU/pseudo-scalar-transcendental.ll | 92 +++++++++++++---------
 2 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 07d79d677104..3281eabcd4ad 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -15189,6 +15189,12 @@ SDValue SITargetLowering::performFMulCombine(SDNode *N,
   EVT ScalarVT = VT.getScalarType();
   EVT IntVT = VT.changeElementType(MVT::i32);
 
+  if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() &&
+      (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) {
+    // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32.
+    return SDValue();
+  }
+
   SDValue LHS = N->getOperand(0);
   SDValue RHS = N->getOperand(1);
 
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index 733fe9317ddc..a2e7f2e62f5d 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -2,19 +2,34 @@
 ; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG %s
 ; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL %s
 
+; TODO: GlobalISel should avoid generating v_ldexp_f32.
 define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
-; GFX12-LABEL: v_s_exp_f32:
-; GFX12:       ; %bb.0:
-; GFX12-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
-; GFX12-NEXT:    s_cselect_b32 s1, 0x42800000, 0
-; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-NEXT:    s_add_f32 s0, s0, s1
-; GFX12-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
-; GFX12-NEXT:    v_s_exp_f32 s0, s0
-; GFX12-NEXT:    s_wait_alu 0xf1ff
-; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT:    v_ldexp_f32 v0, s0, s1
-; GFX12-NEXT:    ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_exp_f32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42800000, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_add_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x1f800000, 1.0
+; GFX12-SDAG-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: v_s_exp_f32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_cmp_lt_f32 s0, 0xc2fc0000
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0x42800000, 0
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    s_add_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_cselect_b32 s1, 0xffffffc0, 0
+; GFX12-GISEL-NEXT:    v_s_exp_f32 s0, s0
+; GFX12-GISEL-NEXT:    s_wait_alu 0xf1ff
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-NEXT:    v_ldexp_f32 v0, s0, s1
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
   %result = call float @llvm.exp2.f32(float %src)
   ret float %result
 }
@@ -59,14 +74,15 @@ define amdgpu_cs float @v_s_log_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: v_s_log_f32:
 ; GFX12-SDAG:       ; %bb.0:
 ; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s0, 0x800000
-; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 32, 0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, s0, s1
-; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
-; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42000000, 0
+; GFX12-SDAG-NEXT:    v_s_log_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_sub_f32 s0, s0, s1
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: v_s_log_f32:
@@ -147,7 +163,7 @@ define amdgpu_cs half @v_s_rcp_f16(half inreg %src) {
   ret half %result
 }
 
-; TODO-GFX12: GlobalISel should generate v_s_rsq.
+; TODO: GlobalISel should generate v_s_rsq.
 define amdgpu_cs float @v_s_rsq_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: v_s_rsq_f32:
 ; GFX12-SDAG:       ; %bb.0:
@@ -184,7 +200,7 @@ define amdgpu_cs half @v_s_rsq_f16(half inreg %src) {
   ret half %result
 }
 
-; TODO-GFX12: Should not use any VALU instructions.
+; TODO: Should avoid generating v_cmp_class_f32.
 define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: v_s_sqrt_f32:
 ; GFX12-SDAG:       ; %bb.0:
@@ -298,16 +314,18 @@ define amdgpu_cs half @v_amdgcn_sqrt_f16(half inreg %src)  {
 define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: srcmods_abs_f32:
 ; GFX12-SDAG:       ; %bb.0:
-; GFX12-SDAG-NEXT:    s_and_b32 s1, s0, 0x7fffffff
+; GFX12-SDAG-NEXT:    s_bitset0_b32 s0, 31
 ; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s1, 0x800000
-; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 32, 0
-; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, |s0|, s1
-; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX12-SDAG-NEXT:    s_cmp_lt_f32 s0, 0x800000
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x4f800000, 1.0
+; GFX12-SDAG-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42000000, 0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_s_log_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_sub_f32 s0, s0, s1
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: srcmods_abs_f32:
@@ -333,15 +351,17 @@ define amdgpu_cs float @srcmods_abs_f32(float inreg %src) {
 define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
 ; GFX12-SDAG-LABEL: srcmods_neg_f32:
 ; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_xor_b32 s1, s0, 0x80000000
 ; GFX12-SDAG-NEXT:    s_cmp_gt_f32 s0, 0x80800000
-; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 32, 0
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-SDAG-NEXT:    v_ldexp_f32 v0, -s0, s1
-; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x42000000, 0
-; GFX12-SDAG-NEXT:    v_log_f32_e32 v0, v0
+; GFX12-SDAG-NEXT:    s_cselect_b32 s0, 0x4f800000, 1.0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_mul_f32 s0, s1, s0
+; GFX12-SDAG-NEXT:    s_cselect_b32 s1, 0x42000000, 0
+; GFX12-SDAG-NEXT:    v_s_log_f32 s0, s0
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
+; GFX12-SDAG-NEXT:    s_sub_f32 s0, s0, s1
 ; GFX12-SDAG-NEXT:    s_wait_alu 0xfffe
-; GFX12-SDAG-NEXT:    s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-NEXT:    v_subrev_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    v_mov_b32_e32 v0, s0
 ; GFX12-SDAG-NEXT:    ; return to shader part epilog
 ;
 ; GFX12-GISEL-LABEL: srcmods_neg_f32:
-- 
cgit v1.2.3


From 749e4a53d252e23e870d4a1638ff9d846af58e7f Mon Sep 17 00:00:00 2001
From: Jonas Devlieghere <jonas@devlieghere.com>
Date: Fri, 20 Jun 2025 10:53:52 -0500
Subject: [lldb] Fix ASCII art in CommandObjectProtocolServer (NFC)

---
 lldb/source/Commands/CommandObjectProtocolServer.cpp | 3 +--
 lldb/source/Commands/CommandObjectProtocolServer.h   | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/lldb/source/Commands/CommandObjectProtocolServer.cpp b/lldb/source/Commands/CommandObjectProtocolServer.cpp
index 420fc5fdddad..115754769f3e 100644
--- a/lldb/source/Commands/CommandObjectProtocolServer.cpp
+++ b/lldb/source/Commands/CommandObjectProtocolServer.cpp
@@ -1,5 +1,4 @@
-//===-- CommandObjectProtocolServer.cpp
-//----------------------------------------------===//
+//===-- CommandObjectProtocolServer.cpp -----------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
diff --git a/lldb/source/Commands/CommandObjectProtocolServer.h b/lldb/source/Commands/CommandObjectProtocolServer.h
index 3591216b014c..791b3a78aaf0 100644
--- a/lldb/source/Commands/CommandObjectProtocolServer.h
+++ b/lldb/source/Commands/CommandObjectProtocolServer.h
@@ -1,5 +1,4 @@
-//===-- CommandObjectProtocolServer.h
-//------------------------------------------------===//
+//===-- CommandObjectProtocolServer.h -------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
-- 
cgit v1.2.3


From 151ee0faad427651304b51b8af77704be26bb485 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 17:03:23 +0100
Subject: [X86] SimplifyDemandedVectorEltsForTargetNode - ensure
 X86ISD::VPERMILPV node use v2f64/v4f32 types

When reducing v4f64/v8f32 non-lane crossing X86ISD::VPERMV nodes, we use X86ISD::VPERMILPV nodes for 128-bits, but these are only available for fp types.

Fixes #145046
---
 llvm/lib/Target/X86/X86ISelLowering.cpp                |  9 +++++++--
 llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll | 11 +++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 085f44e72476..c32737010560 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -44110,8 +44110,13 @@ bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
           // For 128-bit v2X64/v4X32 instructions, use VPERMILPD/VPERMILPS.
           if (VT.is512BitVector() || VT.getScalarSizeInBits() <= 16)
             Ext = TLO.DAG.getNode(Opc, DL, HalfVT, M, V);
-          else
-            Ext = TLO.DAG.getNode(X86ISD::VPERMILPV, DL, HalfVT, V, M);
+          else {
+            MVT ShufSVT = MVT::getFloatingPointVT(VT.getScalarSizeInBits());
+            MVT ShufVT = HalfVT.changeVectorElementType(ShufSVT);
+            Ext = TLO.DAG.getNode(X86ISD::VPERMILPV, DL, ShufVT,
+                                  TLO.DAG.getBitcast(ShufVT, V), M);
+            Ext = TLO.DAG.getBitcast(HalfVT, Ext);
+          }
           SDValue Insert = widenSubVector(Ext, /*ZeroNewElements=*/false,
                                           Subtarget, TLO.DAG, DL, SizeInBits);
           return TLO.CombineTo(Op, Insert);
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
index f4f4842e4c69..f7764b1593b5 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-combining-avx2.ll
@@ -775,6 +775,17 @@ define <32 x i8> @combine_pshufb_pshufb_or_pshufb(<32 x i8> %a0) {
   ret <32 x i8> %4
 }
 
+define <4 x i32> @extract_vpermd(<8 x i32> %a0) {
+; CHECK-LABEL: extract_vpermd:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[1,1,3,0]
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    ret{{[l|q]}}
+  %1 = tail call <8 x i32> @llvm.x86.avx2.permd(<8 x i32> %a0, <8 x i32> <i32 1, i32 1, i32 3, i32 0, i32 1, i32 0, i32 7, i32 6>)
+  %2 = shufflevector <8 x i32> %1, <8 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  ret <4 x i32> %2
+}
+
 ; Not beneficial to concatenate both inputs just to create a 256-bit vpaddb
 define <32 x i8> @concat_add_unnecessary(<16 x i8> %a0, <16 x i8> noundef %a1, <16 x i8> %a2) nounwind {
 ; CHECK-LABEL: concat_add_unnecessary:
-- 
cgit v1.2.3


From 32fc625a3fa27fa325c75b0fc841db4ce8e06805 Mon Sep 17 00:00:00 2001
From: Timm Baeder <tbaeder@redhat.com>
Date: Fri, 20 Jun 2025 18:06:01 +0200
Subject: =?UTF-8?q?Reapply=20"Reapply=20"[clang][bytecode]=20Allocate=20In?=
 =?UTF-8?q?tegralAP=20and=20Floating=20=E2=80=A6=20(#145014)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

…types usi… (#144676)"

This reverts commit 68471d29eed2c49f9b439e505b3f24d387d54f97.

IntegralAP contains a union:
  union {
    uint64_t *Memory = nullptr;
    uint64_t Val;
  };

On 64bit systems, both Memory and Val have the same size. However, on 32
bit system, Val is 64bit and Memory only 32bit. Which means the default
initializer for Memory will only zero half of Val. We fixed this by
zero-initializing Val explicitly in the IntegralAP(unsigned BitWidth)
constructor.


See also the discussion in
https://github.com/llvm/llvm-project/pull/144246
---
 clang/lib/AST/ByteCode/ByteCodeEmitter.cpp         |   3 +-
 clang/lib/AST/ByteCode/Compiler.cpp                | 113 ++++---
 clang/lib/AST/ByteCode/Compiler.h                  |   1 +
 clang/lib/AST/ByteCode/Descriptor.cpp              |   2 +-
 clang/lib/AST/ByteCode/Disasm.cpp                  |  59 ++--
 clang/lib/AST/ByteCode/Floating.h                  | 252 ++++++++++-----
 clang/lib/AST/ByteCode/Integral.h                  |   3 +
 clang/lib/AST/ByteCode/IntegralAP.h                | 236 ++++++++-------
 clang/lib/AST/ByteCode/Interp.cpp                  | 106 ++++++-
 clang/lib/AST/ByteCode/Interp.h                    | 337 +++++++++++++++++----
 clang/lib/AST/ByteCode/InterpBuiltin.cpp           |  55 +++-
 clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp    |   4 +-
 clang/lib/AST/ByteCode/InterpState.h               |  30 ++
 clang/lib/AST/ByteCode/Opcodes.td                  |  14 +-
 clang/lib/AST/ByteCode/PrimType.h                  |  17 ++
 clang/lib/AST/ByteCode/Program.h                   |  24 +-
 .../AST/ByteCode/builtin-bit-cast-long-double.cpp  |  10 +-
 clang/test/AST/ByteCode/builtin-functions.cpp      |  12 +-
 18 files changed, 925 insertions(+), 353 deletions(-)

diff --git a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
index d91d5f16fc7a..965e23503603 100644
--- a/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
+++ b/clang/lib/AST/ByteCode/ByteCodeEmitter.cpp
@@ -176,7 +176,8 @@ static void emitSerialized(std::vector<std::byte> &Code, const T &Val,
   }
 
   // Access must be aligned!
-  size_t ValPos = align(Code.size());
+  assert(aligned(Code.size()));
+  size_t ValPos = Code.size();
   Size = align(Size);
   assert(aligned(ValPos + Size));
   Code.resize(ValPos + Size);
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 9fe4803ce98e..c5ac40210e47 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -748,7 +748,8 @@ bool Compiler<Emitter>::VisitFloatingLiteral(const FloatingLiteral *E) {
   if (DiscardResult)
     return true;
 
-  return this->emitConstFloat(E->getValue(), E);
+  APFloat F = E->getValue();
+  return this->emitFloat(F, E);
 }
 
 template <class Emitter>
@@ -4185,13 +4186,14 @@ bool Compiler<Emitter>::visitZeroInitializer(PrimType T, QualType QT,
                              nullptr, E);
   case PT_MemberPtr:
     return this->emitNullMemberPtr(0, nullptr, E);
-  case PT_Float:
-    return this->emitConstFloat(APFloat::getZero(Ctx.getFloatSemantics(QT)), E);
+  case PT_Float: {
+    APFloat F = APFloat::getZero(Ctx.getFloatSemantics(QT));
+    return this->emitFloat(F, E);
+  }
   case PT_FixedPoint: {
     auto Sem = Ctx.getASTContext().getFixedPointSemantics(E->getType());
     return this->emitConstFixedPoint(FixedPoint::zero(Sem), E);
   }
-    llvm_unreachable("Implement");
   }
   llvm_unreachable("unknown primitive type");
 }
@@ -4674,10 +4676,7 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       if (!visitInitializer(Init))
         return false;
 
-      if (!this->emitFinishInit(Init))
-        return false;
-
-      return this->emitPopPtr(Init);
+      return this->emitFinishInitGlobal(Init);
     };
 
     DeclScope<Emitter> LocalScope(this, VD);
@@ -4698,51 +4697,45 @@ VarCreationState Compiler<Emitter>::visitVarDecl(const VarDecl *VD,
       return false;
 
     return !Init || (checkDecl() && initGlobal(*GlobalIndex));
-  } else {
-    InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
-
-    if (VarT) {
-      unsigned Offset = this->allocateLocalPrimitive(
-          VD, *VarT, VD->getType().isConstQualified(), nullptr,
-          ScopeKind::Block, IsConstexprUnknown);
-      if (Init) {
-        // If this is a toplevel declaration, create a scope for the
-        // initializer.
-        if (Toplevel) {
-          LocalScope<Emitter> Scope(this);
-          if (!this->visit(Init))
-            return false;
-          return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
-        } else {
-          if (!this->visit(Init))
-            return false;
-          return this->emitSetLocal(*VarT, Offset, VD);
-        }
-      }
-    } else {
-      if (std::optional<unsigned> Offset =
-              this->allocateLocal(VD, VD->getType(), nullptr, ScopeKind::Block,
-                                  IsConstexprUnknown)) {
-        if (!Init)
-          return true;
+  }
+  // Local variables.
+  InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
 
-        if (!this->emitGetPtrLocal(*Offset, Init))
+  if (VarT) {
+    unsigned Offset = this->allocateLocalPrimitive(
+        VD, *VarT, VD->getType().isConstQualified(), nullptr, ScopeKind::Block,
+        IsConstexprUnknown);
+    if (Init) {
+      // If this is a toplevel declaration, create a scope for the
+      // initializer.
+      if (Toplevel) {
+        LocalScope<Emitter> Scope(this);
+        if (!this->visit(Init))
           return false;
-
-        if (!visitInitializer(Init))
+        return this->emitSetLocal(*VarT, Offset, VD) && Scope.destroyLocals();
+      } else {
+        if (!this->visit(Init))
           return false;
+        return this->emitSetLocal(*VarT, Offset, VD);
+      }
+    }
+  } else {
+    if (std::optional<unsigned> Offset = this->allocateLocal(
+            VD, VD->getType(), nullptr, ScopeKind::Block, IsConstexprUnknown)) {
+      if (!Init)
+        return true;
 
-        if (!this->emitFinishInit(Init))
-          return false;
+      if (!this->emitGetPtrLocal(*Offset, Init))
+        return false;
 
-        return this->emitPopPtr(Init);
-      }
-      return false;
+      if (!visitInitializer(Init))
+        return false;
+
+      return this->emitFinishInitPop(Init);
     }
-    return true;
+    return false;
   }
-
-  return false;
+  return true;
 }
 
 template <class Emitter>
@@ -4751,8 +4744,10 @@ bool Compiler<Emitter>::visitAPValue(const APValue &Val, PrimType ValType,
   assert(!DiscardResult);
   if (Val.isInt())
     return this->emitConst(Val.getInt(), ValType, E);
-  else if (Val.isFloat())
-    return this->emitConstFloat(Val.getFloat(), E);
+  else if (Val.isFloat()) {
+    APFloat F = Val.getFloat();
+    return this->emitFloat(F, E);
+  }
 
   if (Val.isLValue()) {
     if (Val.isNullPointer())
@@ -6133,8 +6128,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+      APFloat F(TargetSemantics, 1);
+      if (!this->emitFloat(F, E))
         return false;
+
       if (!this->emitAddf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6176,8 +6173,10 @@ bool Compiler<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
       const auto &TargetSemantics = Ctx.getFloatSemantics(E->getType());
       if (!this->emitLoadFloat(E))
         return false;
-      if (!this->emitConstFloat(llvm::APFloat(TargetSemantics, 1), E))
+      APFloat F(TargetSemantics, 1);
+      if (!this->emitFloat(F, E))
         return false;
+
       if (!this->emitSubf(getFPOptions(E), E))
         return false;
       if (!this->emitStoreFloat(E))
@@ -6953,6 +6952,20 @@ bool Compiler<Emitter>::emitDummyPtr(const DeclTy &D, const Expr *E) {
   return true;
 }
 
+template <class Emitter>
+bool Compiler<Emitter>::emitFloat(const APFloat &F, const Expr *E) {
+  assert(!DiscardResult && "Should've been checked before");
+
+  if (Floating::singleWord(F.getSemantics()))
+    return this->emitConstFloat(Floating(F), E);
+
+  APInt I = F.bitcastToAPInt();
+  return this->emitConstFloat(
+      Floating(const_cast<uint64_t *>(I.getRawData()),
+               llvm::APFloatBase::SemanticsToEnum(F.getSemantics())),
+      E);
+}
+
 //  This function is constexpr if and only if To, From, and the types of
 //  all subobjects of To and From are types T such that...
 //  (3.1) - is_union_v<T> is false;
diff --git a/clang/lib/AST/ByteCode/Compiler.h b/clang/lib/AST/ByteCode/Compiler.h
index ac3ad84766dc..a1d068cc7e0a 100644
--- a/clang/lib/AST/ByteCode/Compiler.h
+++ b/clang/lib/AST/ByteCode/Compiler.h
@@ -391,6 +391,7 @@ private:
   bool emitRecordDestruction(const Record *R, SourceInfo Loc);
   bool emitDestruction(const Descriptor *Desc, SourceInfo Loc);
   bool emitDummyPtr(const DeclTy &D, const Expr *E);
+  bool emitFloat(const APFloat &F, const Expr *E);
   unsigned collectBaseOffset(const QualType BaseType,
                              const QualType DerivedType);
   bool emitLambdaStaticInvokerBody(const CXXMethodDecl *MD);
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 5531295dfa2f..46e4d0d940b3 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -368,7 +368,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
                        bool IsTemporary, bool IsConst, UnknownSize)
     : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
       MDSize(MD.value_or(0)),
-      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)),
+      AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
       IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
       IsArray(true), CtorFn(getCtorArrayPrim(Type)),
       DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) {
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index 846dc2fe92a7..a3eecd06369b 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -50,34 +50,57 @@ inline static std::string printArg(Program &P, CodePtr &OpPC) {
 }
 
 template <> inline std::string printArg<Floating>(Program &P, CodePtr &OpPC) {
-  auto F = Floating::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
+  auto Sem = Floating::deserializeSemantics(*OpPC);
 
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  unsigned BitWidth = llvm::APFloatBase::semanticsSizeInBits(
+      llvm::APFloatBase::EnumToSemantics(Sem));
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
+  Floating Result(Memory.get(), Sem);
+  Floating::deserialize(*OpPC, &Result);
+
+  OpPC += align(Result.bytesToSerialize());
+
+  std::string S;
+  llvm::raw_string_ostream SS(S);
+  SS << Result;
+  return S;
 }
 
 template <>
 inline std::string printArg<IntegralAP<false>>(Program &P, CodePtr &OpPC) {
-  auto F = IntegralAP<false>::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
+  using T = IntegralAP<false>;
+  uint32_t BitWidth = T::deserializeSize(*OpPC);
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
 
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  T Result(Memory.get(), BitWidth);
+  T::deserialize(*OpPC, &Result);
+
+  OpPC += align(Result.bytesToSerialize());
+
+  std::string Str;
+  llvm::raw_string_ostream SS(Str);
+  SS << Result;
+  return Str;
 }
+
 template <>
 inline std::string printArg<IntegralAP<true>>(Program &P, CodePtr &OpPC) {
-  auto F = IntegralAP<true>::deserialize(*OpPC);
-  OpPC += align(F.bytesToSerialize());
+  using T = IntegralAP<true>;
+  uint32_t BitWidth = T::deserializeSize(*OpPC);
+  auto Memory =
+      std::make_unique<uint64_t[]>(llvm::APInt::getNumWords(BitWidth));
 
-  std::string Result;
-  llvm::raw_string_ostream SS(Result);
-  SS << F;
-  return Result;
+  T Result(Memory.get(), BitWidth);
+  T::deserialize(*OpPC, &Result);
+
+  OpPC += align(Result.bytesToSerialize());
+
+  std::string Str;
+  llvm::raw_string_ostream SS(Str);
+  SS << Result;
+  return Str;
 }
 
 template <> inline std::string printArg<FixedPoint>(Program &P, CodePtr &OpPC) {
diff --git a/clang/lib/AST/ByteCode/Floating.h b/clang/lib/AST/ByteCode/Floating.h
index 3750568fc23c..659892e720ab 100644
--- a/clang/lib/AST/ByteCode/Floating.h
+++ b/clang/lib/AST/ByteCode/Floating.h
@@ -17,63 +17,79 @@
 #include "clang/AST/APValue.h"
 #include "llvm/ADT/APFloat.h"
 
+// XXX This is just a debugging help. Setting this to 1 will heap-allocate ALL
+// floating values.
+#define ALLOCATE_ALL 0
+
 namespace clang {
 namespace interp {
 
 using APFloat = llvm::APFloat;
 using APSInt = llvm::APSInt;
+using APInt = llvm::APInt;
 
+/// If a Floating is constructed from Memory, it DOES NOT OWN THAT MEMORY.
+/// It will NOT copy the memory (unless, of course, copy() is called) and it
+/// won't alllocate anything. The allocation should happen via InterpState or
+/// Program.
 class Floating final {
 private:
-  // The underlying value storage.
-  APFloat F;
+  union {
+    uint64_t Val = 0;
+    uint64_t *Memory;
+  };
+  llvm::APFloatBase::Semantics Semantics;
+
+  APFloat getValue() const {
+    unsigned BitWidth = bitWidth();
+    if (singleWord())
+      return APFloat(getSemantics(), APInt(BitWidth, Val));
+    unsigned NumWords = numWords();
+    return APFloat(getSemantics(), APInt(BitWidth, NumWords, Memory));
+  }
 
 public:
-  /// Zero-initializes a Floating.
-  Floating() : F(0.0f) {}
-  Floating(const APFloat &F) : F(F) {}
+  Floating() = default;
+  Floating(llvm::APFloatBase::Semantics Semantics)
+      : Val(0), Semantics(Semantics) {}
+  Floating(const APFloat &F) {
 
-  // Static constructors for special floating point values.
-  static Floating getInf(const llvm::fltSemantics &Sem) {
-    return Floating(APFloat::getInf(Sem));
+    Semantics = llvm::APFloatBase::SemanticsToEnum(F.getSemantics());
+    this->copy(F);
   }
-  const APFloat &getAPFloat() const { return F; }
+  Floating(uint64_t *Memory, llvm::APFloatBase::Semantics Semantics)
+      : Memory(Memory), Semantics(Semantics) {}
+
+  APFloat getAPFloat() const { return getValue(); }
 
-  bool operator<(Floating RHS) const { return F < RHS.F; }
-  bool operator>(Floating RHS) const { return F > RHS.F; }
-  bool operator<=(Floating RHS) const { return F <= RHS.F; }
-  bool operator>=(Floating RHS) const { return F >= RHS.F; }
-  bool operator==(Floating RHS) const { return F == RHS.F; }
-  bool operator!=(Floating RHS) const { return F != RHS.F; }
-  Floating operator-() const { return Floating(-F); }
+  bool operator<(Floating RHS) const { return getValue() < RHS.getValue(); }
+  bool operator>(Floating RHS) const { return getValue() > RHS.getValue(); }
+  bool operator<=(Floating RHS) const { return getValue() <= RHS.getValue(); }
+  bool operator>=(Floating RHS) const { return getValue() >= RHS.getValue(); }
 
   APFloat::opStatus convertToInteger(APSInt &Result) const {
     bool IsExact;
-    return F.convertToInteger(Result, llvm::APFloat::rmTowardZero, &IsExact);
+    return getValue().convertToInteger(Result, llvm::APFloat::rmTowardZero,
+                                       &IsExact);
   }
 
-  Floating toSemantics(const llvm::fltSemantics *Sem,
-                       llvm::RoundingMode RM) const {
-    APFloat Copy = F;
+  void toSemantics(const llvm::fltSemantics *Sem, llvm::RoundingMode RM,
+                   Floating *Result) const {
+    APFloat Copy = getValue();
     bool LosesInfo;
     Copy.convert(*Sem, RM, &LosesInfo);
     (void)LosesInfo;
-    return Floating(Copy);
-  }
-
-  /// Convert this Floating to one with the same semantics as \Other.
-  Floating toSemantics(const Floating &Other, llvm::RoundingMode RM) const {
-    return toSemantics(&Other.F.getSemantics(), RM);
+    Result->copy(Copy);
   }
 
   APSInt toAPSInt(unsigned NumBits = 0) const {
-    return APSInt(F.bitcastToAPInt());
+    return APSInt(getValue().bitcastToAPInt());
   }
-  APValue toAPValue(const ASTContext &) const { return APValue(F); }
+  APValue toAPValue(const ASTContext &) const { return APValue(getValue()); }
   void print(llvm::raw_ostream &OS) const {
     // Can't use APFloat::print() since it appends a newline.
     SmallVector<char, 16> Buffer;
-    F.toString(Buffer);
+    getValue().toString(Buffer);
     OS << Buffer;
   }
   std::string toDiagnosticString(const ASTContext &Ctx) const {
@@ -83,25 +99,62 @@ public:
     return NameStr;
   }
 
-  unsigned bitWidth() const { return F.semanticsSizeInBits(F.getSemantics()); }
+  unsigned bitWidth() const {
+    return llvm::APFloatBase::semanticsSizeInBits(getSemantics());
+  }
+  unsigned numWords() const { return llvm::APInt::getNumWords(bitWidth()); }
+  bool singleWord() const {
+#if ALLOCATE_ALL
+    return false;
+#endif
+    return numWords() == 1;
+  }
+  static bool singleWord(const llvm::fltSemantics &Sem) {
+#if ALLOCATE_ALL
+    return false;
+#endif
+    return APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem)) == 1;
+  }
+  const llvm::fltSemantics &getSemantics() const {
+    return llvm::APFloatBase::EnumToSemantics(Semantics);
+  }
+
+  void copy(const APFloat &F) {
+    if (singleWord()) {
+      Val = F.bitcastToAPInt().getZExtValue();
+    } else {
+      assert(Memory);
+      std::memcpy(Memory, F.bitcastToAPInt().getRawData(),
+                  numWords() * sizeof(uint64_t));
+    }
+  }
+
+  void take(uint64_t *NewMemory) {
+    if (singleWord())
+      return;
+
+    if (Memory)
+      std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
+    Memory = NewMemory;
+  }
 
   bool isSigned() const { return true; }
-  bool isNegative() const { return F.isNegative(); }
-  bool isZero() const { return F.isZero(); }
-  bool isNonZero() const { return F.isNonZero(); }
-  bool isMin() const { return F.isSmallest(); }
-  bool isMinusOne() const { return F.isExactlyValue(-1.0); }
-  bool isNan() const { return F.isNaN(); }
-  bool isSignaling() const { return F.isSignaling(); }
-  bool isInf() const { return F.isInfinity(); }
-  bool isFinite() const { return F.isFinite(); }
-  bool isNormal() const { return F.isNormal(); }
-  bool isDenormal() const { return F.isDenormal(); }
-  llvm::FPClassTest classify() const { return F.classify(); }
-  APFloat::fltCategory getCategory() const { return F.getCategory(); }
+  bool isNegative() const { return getValue().isNegative(); }
+  bool isZero() const { return getValue().isZero(); }
+  bool isNonZero() const { return getValue().isNonZero(); }
+  bool isMin() const { return getValue().isSmallest(); }
+  bool isMinusOne() const { return getValue().isExactlyValue(-1.0); }
+  bool isNan() const { return getValue().isNaN(); }
+  bool isSignaling() const { return getValue().isSignaling(); }
+  bool isInf() const { return getValue().isInfinity(); }
+  bool isFinite() const { return getValue().isFinite(); }
+  bool isNormal() const { return getValue().isNormal(); }
+  bool isDenormal() const { return getValue().isDenormal(); }
+  llvm::FPClassTest classify() const { return getValue().classify(); }
+  APFloat::fltCategory getCategory() const { return getValue().getCategory(); }
 
   ComparisonCategoryResult compare(const Floating &RHS) const {
-    llvm::APFloatBase::cmpResult CmpRes = F.compare(RHS.F);
+    llvm::APFloatBase::cmpResult CmpRes = getValue().compare(RHS.getValue());
     switch (CmpRes) {
     case llvm::APFloatBase::cmpLessThan:
       return ComparisonCategoryResult::Less;
@@ -118,97 +171,130 @@ public:
   static APFloat::opStatus fromIntegral(APSInt Val,
                                         const llvm::fltSemantics &Sem,
                                         llvm::RoundingMode RM,
-                                        Floating &Result) {
+                                        Floating *Result) {
     APFloat F = APFloat(Sem);
     APFloat::opStatus Status = F.convertFromAPInt(Val, Val.isSigned(), RM);
-    Result = Floating(F);
+    Result->copy(F);
     return Status;
   }
 
-  static Floating bitcastFromMemory(const std::byte *Buff,
-                                    const llvm::fltSemantics &Sem) {
+  static void bitcastFromMemory(const std::byte *Buff,
+                                const llvm::fltSemantics &Sem,
+                                Floating *Result) {
     size_t Size = APFloat::semanticsSizeInBits(Sem);
     llvm::APInt API(Size, true);
     llvm::LoadIntFromMemory(API, (const uint8_t *)Buff, Size / 8);
-
-    return Floating(APFloat(Sem, API));
+    Result->copy(APFloat(Sem, API));
   }
 
   void bitcastToMemory(std::byte *Buff) const {
-    llvm::APInt API = F.bitcastToAPInt();
+    llvm::APInt API = getValue().bitcastToAPInt();
     llvm::StoreIntToMemory(API, (uint8_t *)Buff, bitWidth() / 8);
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    return sizeof(llvm::fltSemantics *) +
-           (APFloat::semanticsSizeInBits(F.getSemantics()) / 8);
+    return sizeof(Semantics) + (numWords() * sizeof(uint64_t));
   }
 
   void serialize(std::byte *Buff) const {
-    // Semantics followed by an APInt.
-    *reinterpret_cast<const llvm::fltSemantics **>(Buff) = &F.getSemantics();
-
-    llvm::APInt API = F.bitcastToAPInt();
-    llvm::StoreIntToMemory(API, (uint8_t *)(Buff + sizeof(void *)),
-                           bitWidth() / 8);
+    std::memcpy(Buff, &Semantics, sizeof(Semantics));
+    if (singleWord()) {
+      std::memcpy(Buff + sizeof(Semantics), &Val, sizeof(uint64_t));
+    } else {
+      std::memcpy(Buff + sizeof(Semantics), Memory,
+                  numWords() * sizeof(uint64_t));
+    }
   }
 
-  static Floating deserialize(const std::byte *Buff) {
-    const llvm::fltSemantics *Sem;
-    std::memcpy((void *)&Sem, Buff, sizeof(void *));
-    return bitcastFromMemory(Buff + sizeof(void *), *Sem);
+  static llvm::APFloatBase::Semantics
+  deserializeSemantics(const std::byte *Buff) {
+    return *reinterpret_cast<const llvm::APFloatBase::Semantics *>(Buff);
   }
 
-  static Floating abs(const Floating &F) {
-    APFloat V = F.F;
-    if (V.isNegative())
-      V.changeSign();
-    return Floating(V);
+  static void deserialize(const std::byte *Buff, Floating *Result) {
+    llvm::APFloatBase::Semantics Semantics;
+    std::memcpy(&Semantics, Buff, sizeof(Semantics));
+
+    unsigned BitWidth = llvm::APFloat::semanticsSizeInBits(
+        llvm::APFloatBase::EnumToSemantics(Semantics));
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
+
+    Result->Semantics = Semantics;
+    if (NumWords == 1 && !ALLOCATE_ALL) {
+      std::memcpy(&Result->Val, Buff + sizeof(Semantics), sizeof(uint64_t));
+    } else {
+      assert(Result->Memory);
+      std::memcpy(Result->Memory, Buff + sizeof(Semantics),
+                  NumWords * sizeof(uint64_t));
+    }
   }
 
   // -------
 
   static APFloat::opStatus add(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.add(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.add(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus increment(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.F.getSemantics(), 1);
-    *R = Floating(A.F);
-    return R->F.add(One, RM);
+    APFloat One(A.getSemantics(), 1);
+    APFloat LHS = A.getValue();
+
+    auto Status = LHS.add(One, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus sub(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.subtract(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.subtract(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus decrement(const Floating &A, llvm::RoundingMode RM,
                                      Floating *R) {
-    APFloat One(A.F.getSemantics(), 1);
-    *R = Floating(A.F);
-    return R->F.subtract(One, RM);
+    APFloat One(A.getSemantics(), 1);
+    APFloat LHS = A.getValue();
+
+    auto Status = LHS.subtract(One, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus mul(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.multiply(B.F, RM);
+
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.multiply(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static APFloat::opStatus div(const Floating &A, const Floating &B,
                                llvm::RoundingMode RM, Floating *R) {
-    *R = Floating(A.F);
-    return R->F.divide(B.F, RM);
+    APFloat LHS = A.getValue();
+    APFloat RHS = B.getValue();
+
+    auto Status = LHS.divide(RHS, RM);
+    R->copy(LHS);
+    return Status;
   }
 
   static bool neg(const Floating &A, Floating *R) {
-    *R = -A;
+    R->copy(-A.getValue());
     return false;
   }
 };
diff --git a/clang/lib/AST/ByteCode/Integral.h b/clang/lib/AST/ByteCode/Integral.h
index 13fdb5369f2b..af5cd2d13ecc 100644
--- a/clang/lib/AST/ByteCode/Integral.h
+++ b/clang/lib/AST/ByteCode/Integral.h
@@ -99,6 +99,9 @@ public:
   bool operator>=(Integral RHS) const { return V >= RHS.V; }
   bool operator==(Integral RHS) const { return V == RHS.V; }
   bool operator!=(Integral RHS) const { return V != RHS.V; }
+  bool operator>=(unsigned RHS) const {
+    return static_cast<unsigned>(V) >= RHS;
+  }
 
   bool operator>(unsigned RHS) const {
     return V >= 0 && static_cast<unsigned>(V) > RHS;
diff --git a/clang/lib/AST/ByteCode/IntegralAP.h b/clang/lib/AST/ByteCode/IntegralAP.h
index 8ee08dfb5cfe..316c003e0e50 100644
--- a/clang/lib/AST/ByteCode/IntegralAP.h
+++ b/clang/lib/AST/ByteCode/IntegralAP.h
@@ -28,12 +28,19 @@ namespace interp {
 
 using APInt = llvm::APInt;
 using APSInt = llvm::APSInt;
-template <unsigned Bits, bool Signed> class Integral;
 
+/// If an IntegralAP is constructed from Memory, it DOES NOT OWN THAT MEMORY.
+/// It will NOT copy the memory (unless, of course, copy() is called) and it
+/// won't alllocate anything. The allocation should happen via InterpState or
+/// Program.
 template <bool Signed> class IntegralAP final {
-private:
+public:
+  union {
+    uint64_t *Memory = nullptr;
+    uint64_t Val;
+  };
+  uint32_t BitWidth = 0;
   friend IntegralAP<!Signed>;
-  APInt V;
 
   template <typename T, bool InputSigned>
   static T truncateCast(const APInt &V) {
@@ -52,106 +59,118 @@ private:
                                : V.trunc(BitSize).getZExtValue();
   }
 
+  APInt getValue() const {
+    if (singleWord())
+      return APInt(BitWidth, Val, Signed);
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
+    return llvm::APInt(BitWidth, NumWords, Memory);
+  }
+
 public:
   using AsUnsigned = IntegralAP<false>;
 
-  template <typename T>
-  IntegralAP(T Value, unsigned BitWidth)
-      : V(APInt(BitWidth, static_cast<uint64_t>(Value), Signed)) {}
+  void take(uint64_t *NewMemory) {
+    assert(!singleWord());
+    std::memcpy(NewMemory, Memory, numWords() * sizeof(uint64_t));
+    Memory = NewMemory;
+  }
 
-  IntegralAP(APInt V) : V(V) {}
-  /// Arbitrary value for uninitialized variables.
-  IntegralAP() : IntegralAP(Signed ? -1 : 7, 3) {}
+  void copy(const APInt &V) {
+    assert(BitWidth == V.getBitWidth());
+    assert(numWords() == V.getNumWords());
 
-  IntegralAP operator-() const { return IntegralAP(-V); }
-  IntegralAP operator-(const IntegralAP &Other) const {
-    return IntegralAP(V - Other.V);
+    if (V.isSingleWord()) {
+      if constexpr (Signed)
+        Val = V.getSExtValue();
+      else
+        Val = V.getZExtValue();
+      return;
+    }
+    assert(Memory);
+    std::memcpy(Memory, V.getRawData(), V.getNumWords() * sizeof(uint64_t));
+  }
+
+  IntegralAP() = default;
+  /// Zeroed, single-word IntegralAP of the given bitwidth.
+  IntegralAP(unsigned BitWidth) : Val(0), BitWidth(BitWidth) {
+    assert(singleWord());
+  }
+  IntegralAP(uint64_t *Memory, unsigned BitWidth)
+      : Memory(Memory), BitWidth(BitWidth) {}
+  IntegralAP(const APInt &V) : BitWidth(V.getBitWidth()) {
+    if (V.isSingleWord()) {
+      Val = Signed ? V.getSExtValue() : V.getZExtValue();
+    } else {
+      Memory = const_cast<uint64_t *>(V.getRawData());
+    }
   }
+
+  IntegralAP operator-() const { return IntegralAP(-getValue()); }
   bool operator>(const IntegralAP &RHS) const {
     if constexpr (Signed)
-      return V.ugt(RHS.V);
-    return V.sgt(RHS.V);
+      return getValue().sgt(RHS.getValue());
+    return getValue().ugt(RHS.getValue());
   }
-  bool operator>=(IntegralAP RHS) const {
+  bool operator>=(unsigned RHS) const {
     if constexpr (Signed)
-      return V.uge(RHS.V);
-    return V.sge(RHS.V);
+      return getValue().sge(RHS);
+    return getValue().uge(RHS);
   }
   bool operator<(IntegralAP RHS) const {
     if constexpr (Signed)
-      return V.slt(RHS.V);
-    return V.slt(RHS.V);
-  }
-  bool operator<=(IntegralAP RHS) const {
-    if constexpr (Signed)
-      return V.ult(RHS.V);
-    return V.ult(RHS.V);
+      return getValue().slt(RHS.getValue());
+    return getValue().ult(RHS.getValue());
   }
 
   template <typename Ty, typename = std::enable_if_t<std::is_integral_v<Ty>>>
   explicit operator Ty() const {
-    return truncateCast<Ty, Signed>(V);
+    return truncateCast<Ty, Signed>(getValue());
   }
 
   template <typename T> static IntegralAP from(T Value, unsigned NumBits = 0) {
+    if (NumBits == 0)
+      NumBits = sizeof(T) * 8;
     assert(NumBits > 0);
     APInt Copy = APInt(NumBits, static_cast<uint64_t>(Value), Signed);
-
+    assert(false);
     return IntegralAP<Signed>(Copy);
   }
 
-  template <bool InputSigned>
-  static IntegralAP from(IntegralAP<InputSigned> V, unsigned NumBits = 0) {
-    if (NumBits == 0)
-      NumBits = V.bitWidth();
-
-    if constexpr (InputSigned)
-      return IntegralAP<Signed>(V.V.sextOrTrunc(NumBits));
-    return IntegralAP<Signed>(V.V.zextOrTrunc(NumBits));
-  }
-
-  template <unsigned Bits, bool InputSigned>
-  static IntegralAP from(Integral<Bits, InputSigned> I, unsigned BitWidth) {
-    return IntegralAP<Signed>(I.toAPInt(BitWidth));
-  }
-
-  static IntegralAP zero(int32_t BitWidth) {
-    APInt V = APInt(BitWidth, 0LL, Signed);
-    return IntegralAP(V);
-  }
-
-  constexpr unsigned bitWidth() const { return V.getBitWidth(); }
+  constexpr uint32_t bitWidth() const { return BitWidth; }
+  constexpr unsigned numWords() const { return APInt::getNumWords(BitWidth); }
+  constexpr bool singleWord() const { return numWords() == 1; }
 
   APSInt toAPSInt(unsigned Bits = 0) const {
     if (Bits == 0)
       Bits = bitWidth();
 
+    APInt V = getValue();
     if constexpr (Signed)
-      return APSInt(V.sext(Bits), !Signed);
+      return APSInt(getValue().sext(Bits), !Signed);
     else
-      return APSInt(V.zext(Bits), !Signed);
+      return APSInt(getValue().zext(Bits), !Signed);
   }
   APValue toAPValue(const ASTContext &) const { return APValue(toAPSInt()); }
 
-  bool isZero() const { return V.isZero(); }
+  bool isZero() const { return getValue().isZero(); }
   bool isPositive() const {
     if constexpr (Signed)
-      return V.isNonNegative();
+      return getValue().isNonNegative();
     return true;
   }
   bool isNegative() const {
     if constexpr (Signed)
-      return !V.isNonNegative();
+      return !getValue().isNonNegative();
     return false;
   }
-  bool isMin() const { return V.isMinValue(); }
-  bool isMax() const { return V.isMaxValue(); }
+  bool isMin() const { return getValue().isMinValue(); }
+  bool isMax() const { return getValue().isMaxValue(); }
   static constexpr bool isSigned() { return Signed; }
-  bool isMinusOne() const { return Signed && V == -1; }
+  bool isMinusOne() const { return Signed && getValue().isAllOnes(); }
 
-  unsigned countLeadingZeros() const { return V.countl_zero(); }
+  unsigned countLeadingZeros() const { return getValue().countl_zero(); }
 
-  void print(llvm::raw_ostream &OS) const { V.print(OS, Signed);}
+  void print(llvm::raw_ostream &OS) const { getValue().print(OS, Signed); }
   std::string toDiagnosticString(const ASTContext &Ctx) const {
     std::string NameStr;
     llvm::raw_string_ostream OS(NameStr);
@@ -161,53 +180,57 @@ public:
 
   IntegralAP truncate(unsigned BitWidth) const {
     if constexpr (Signed)
-      return IntegralAP(V.trunc(BitWidth).sextOrTrunc(this->bitWidth()));
+      return IntegralAP(
+          getValue().trunc(BitWidth).sextOrTrunc(this->bitWidth()));
     else
-      return IntegralAP(V.trunc(BitWidth).zextOrTrunc(this->bitWidth()));
+      return IntegralAP(
+          getValue().trunc(BitWidth).zextOrTrunc(this->bitWidth()));
   }
 
   IntegralAP<false> toUnsigned() const {
-    APInt Copy = V;
-    return IntegralAP<false>(Copy);
+    return IntegralAP<false>(Memory, BitWidth);
   }
 
   void bitcastToMemory(std::byte *Dest) const {
-    llvm::StoreIntToMemory(V, (uint8_t *)Dest, bitWidth() / 8);
+    llvm::StoreIntToMemory(getValue(), (uint8_t *)Dest, bitWidth() / 8);
   }
 
-  static IntegralAP bitcastFromMemory(const std::byte *Src, unsigned BitWidth) {
+  static void bitcastFromMemory(const std::byte *Src, unsigned BitWidth,
+                                IntegralAP *Result) {
     APInt V(BitWidth, static_cast<uint64_t>(0), Signed);
     llvm::LoadIntFromMemory(V, (const uint8_t *)Src, BitWidth / 8);
-    return IntegralAP(V);
+    Result->copy(V);
   }
 
   ComparisonCategoryResult compare(const IntegralAP &RHS) const {
     assert(Signed == RHS.isSigned());
     assert(bitWidth() == RHS.bitWidth());
+    APInt V1 = getValue();
+    APInt V2 = RHS.getValue();
     if constexpr (Signed) {
-      if (V.slt(RHS.V))
+      if (V1.slt(V2))
         return ComparisonCategoryResult::Less;
-      if (V.sgt(RHS.V))
+      if (V1.sgt(V2))
         return ComparisonCategoryResult::Greater;
       return ComparisonCategoryResult::Equal;
     }
 
     assert(!Signed);
-    if (V.ult(RHS.V))
+    if (V1.ult(V2))
       return ComparisonCategoryResult::Less;
-    if (V.ugt(RHS.V))
+    if (V1.ugt(V2))
       return ComparisonCategoryResult::Greater;
     return ComparisonCategoryResult::Equal;
   }
 
   static bool increment(IntegralAP A, IntegralAP *R) {
-    IntegralAP<Signed> One(1, A.bitWidth());
-    return add(A, One, A.bitWidth() + 1, R);
+    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
+    return add(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
   }
 
   static bool decrement(IntegralAP A, IntegralAP *R) {
-    IntegralAP<Signed> One(1, A.bitWidth());
-    return sub(A, One, A.bitWidth() + 1, R);
+    APSInt One(APInt(A.bitWidth(), 1ull, Signed), !Signed);
+    return sub(A, IntegralAP<Signed>(One), A.bitWidth() + 1, R);
   }
 
   static bool add(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
@@ -224,87 +247,96 @@ public:
 
   static bool rem(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      *R = IntegralAP(A.V.srem(B.V));
+      R->copy(A.getValue().srem(B.getValue()));
     else
-      *R = IntegralAP(A.V.urem(B.V));
+      R->copy(A.getValue().urem(B.getValue()));
     return false;
   }
 
   static bool div(IntegralAP A, IntegralAP B, unsigned OpBits, IntegralAP *R) {
     if constexpr (Signed)
-      *R = IntegralAP(A.V.sdiv(B.V));
+      R->copy(A.getValue().sdiv(B.getValue()));
     else
-      *R = IntegralAP(A.V.udiv(B.V));
+      R->copy(A.getValue().udiv(B.getValue()));
     return false;
   }
 
   static bool bitAnd(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    *R = IntegralAP(A.V & B.V);
+    R->copy(A.getValue() & B.getValue());
     return false;
   }
 
   static bool bitOr(IntegralAP A, IntegralAP B, unsigned OpBits,
                     IntegralAP *R) {
-    *R = IntegralAP(A.V | B.V);
+    R->copy(A.getValue() | B.getValue());
     return false;
   }
 
   static bool bitXor(IntegralAP A, IntegralAP B, unsigned OpBits,
                      IntegralAP *R) {
-    *R = IntegralAP(A.V ^ B.V);
+    R->copy(A.getValue() ^ B.getValue());
     return false;
   }
 
   static bool neg(const IntegralAP &A, IntegralAP *R) {
-    APInt AI = A.V;
+    APInt AI = A.getValue();
     AI.negate();
-    *R = IntegralAP(AI);
+    R->copy(AI);
     return false;
   }
 
   static bool comp(IntegralAP A, IntegralAP *R) {
-    *R = IntegralAP(~A.V);
+    R->copy(~A.getValue());
     return false;
   }
 
   static void shiftLeft(const IntegralAP A, const IntegralAP B, unsigned OpBits,
                         IntegralAP *R) {
-    *R = IntegralAP(A.V.shl(B.V.getZExtValue()));
+    *R = IntegralAP(A.getValue().shl(B.getValue().getZExtValue()));
   }
 
   static void shiftRight(const IntegralAP A, const IntegralAP B,
                          unsigned OpBits, IntegralAP *R) {
-    unsigned ShiftAmount = B.V.getZExtValue();
+    unsigned ShiftAmount = B.getValue().getZExtValue();
     if constexpr (Signed)
-      *R = IntegralAP(A.V.ashr(ShiftAmount));
+      R->copy(A.getValue().ashr(ShiftAmount));
     else
-      *R = IntegralAP(A.V.lshr(ShiftAmount));
+      R->copy(A.getValue().lshr(ShiftAmount));
   }
 
   // === Serialization support ===
   size_t bytesToSerialize() const {
-    // 4 bytes for the BitWidth followed by N bytes for the actual APInt.
-    return sizeof(uint32_t) + (V.getBitWidth() / CHAR_BIT);
+    assert(BitWidth != 0);
+    return sizeof(uint32_t) + (numWords() * sizeof(uint64_t));
   }
 
   void serialize(std::byte *Buff) const {
-    assert(V.getBitWidth() < std::numeric_limits<uint8_t>::max());
-    uint32_t BitWidth = V.getBitWidth();
-
     std::memcpy(Buff, &BitWidth, sizeof(uint32_t));
-    llvm::StoreIntToMemory(V, (uint8_t *)(Buff + sizeof(uint32_t)),
-                           BitWidth / CHAR_BIT);
+    if (singleWord())
+      std::memcpy(Buff + sizeof(uint32_t), &Val, sizeof(uint64_t));
+    else {
+      std::memcpy(Buff + sizeof(uint32_t), Memory,
+                  numWords() * sizeof(uint64_t));
+    }
   }
 
-  static IntegralAP<Signed> deserialize(const std::byte *Buff) {
-    uint32_t BitWidth;
-    std::memcpy(&BitWidth, Buff, sizeof(uint32_t));
-    IntegralAP<Signed> Val(APInt(BitWidth, 0ull, !Signed));
+  static uint32_t deserializeSize(const std::byte *Buff) {
+    return *reinterpret_cast<const uint32_t *>(Buff);
+  }
+
+  static void deserialize(const std::byte *Buff, IntegralAP<Signed> *Result) {
+    uint32_t BitWidth = Result->BitWidth;
+    assert(BitWidth != 0);
+    unsigned NumWords = llvm::APInt::getNumWords(BitWidth);
 
-    llvm::LoadIntFromMemory(Val.V, (const uint8_t *)Buff + sizeof(uint32_t),
-                            BitWidth / CHAR_BIT);
-    return Val;
+    if (NumWords == 1)
+      std::memcpy(&Result->Val, Buff + sizeof(uint32_t), sizeof(uint64_t));
+    else {
+      assert(Result->Memory);
+      std::memcpy(Result->Memory, Buff + sizeof(uint32_t),
+                  NumWords * sizeof(uint64_t));
+    }
   }
 
 private:
@@ -312,7 +344,7 @@ private:
   static bool CheckAddSubMulUB(const IntegralAP &A, const IntegralAP &B,
                                unsigned BitWidth, IntegralAP *R) {
     if constexpr (!Signed) {
-      R->V = Op<APInt>{}(A.V, B.V);
+      R->copy(Op<APInt>{}(A.getValue(), B.getValue()));
       return false;
     }
 
@@ -320,7 +352,7 @@ private:
     const APSInt &RHS = B.toAPSInt();
     APSInt Value = Op<APSInt>{}(LHS.extend(BitWidth), RHS.extend(BitWidth));
     APSInt Result = Value.trunc(LHS.getBitWidth());
-    R->V = Result;
+    R->copy(Result);
 
     return Result.extend(BitWidth) != Value;
   }
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index 5c8abffb3a99..1e2032feabb6 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1935,8 +1935,10 @@ bool CastPointerIntegralAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  S.Stk.push<IntegralAP<false>>(
-      IntegralAP<false>::from(Ptr.getIntegerRepresentation(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
+
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
@@ -1946,8 +1948,10 @@ bool CastPointerIntegralAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
   if (!CheckPointerToIntegralCast(S, OpPC, Ptr, BitWidth))
     return false;
 
-  S.Stk.push<IntegralAP<true>>(
-      IntegralAP<true>::from(Ptr.getIntegerRepresentation(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  Result.copy(APInt(BitWidth, Ptr.getIntegerRepresentation()));
+
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2053,6 +2057,100 @@ bool arePotentiallyOverlappingStringLiterals(const Pointer &LHS,
   return Shorter == Longer.take_front(Shorter.size());
 }
 
+static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr,
+                                PrimType T) {
+
+  if (T == PT_IntAPS) {
+    auto &Val = Ptr.deref<IntegralAP<true>>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  } else if (T == PT_IntAP) {
+    auto &Val = Ptr.deref<IntegralAP<false>>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  } else if (T == PT_Float) {
+    auto &Val = Ptr.deref<Floating>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  }
+}
+
+template <typename T>
+static void copyPrimitiveMemory(InterpState &S, const Pointer &Ptr) {
+  assert(needsAlloc<T>());
+  auto &Val = Ptr.deref<T>();
+  if (!Val.singleWord()) {
+    uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+    Val.take(NewMemory);
+  }
+}
+
+static void finishGlobalRecurse(InterpState &S, const Pointer &Ptr) {
+  if (const Record *R = Ptr.getRecord()) {
+    for (const Record::Field &Fi : R->fields()) {
+      if (Fi.Desc->isPrimitive()) {
+        TYPE_SWITCH_ALLOC(Fi.Desc->getPrimType(), {
+          copyPrimitiveMemory<T>(S, Ptr.atField(Fi.Offset));
+        });
+        copyPrimitiveMemory(S, Ptr.atField(Fi.Offset), Fi.Desc->getPrimType());
+      } else
+        finishGlobalRecurse(S, Ptr.atField(Fi.Offset));
+    }
+    return;
+  }
+
+  if (const Descriptor *D = Ptr.getFieldDesc(); D && D->isArray()) {
+    unsigned NumElems = D->getNumElems();
+    if (NumElems == 0)
+      return;
+
+    if (D->isPrimitiveArray()) {
+      PrimType PT = D->getPrimType();
+      if (!needsAlloc(PT))
+        return;
+      assert(NumElems >= 1);
+      const Pointer EP = Ptr.atIndex(0);
+      bool AllSingleWord = true;
+      TYPE_SWITCH_ALLOC(PT, {
+        if (!EP.deref<T>().singleWord()) {
+          copyPrimitiveMemory<T>(S, EP);
+          AllSingleWord = false;
+        }
+      });
+      if (AllSingleWord)
+        return;
+      for (unsigned I = 1; I != D->getNumElems(); ++I) {
+        const Pointer EP = Ptr.atIndex(I);
+        copyPrimitiveMemory(S, EP, PT);
+      }
+    } else {
+      assert(D->isCompositeArray());
+      for (unsigned I = 0; I != D->getNumElems(); ++I) {
+        const Pointer EP = Ptr.atIndex(I).narrow();
+        finishGlobalRecurse(S, EP);
+      }
+    }
+  }
+}
+
+bool FinishInitGlobal(InterpState &S, CodePtr OpPC) {
+  const Pointer &Ptr = S.Stk.pop<Pointer>();
+
+  finishGlobalRecurse(S, Ptr);
+  if (Ptr.canBeInitialized()) {
+    Ptr.initialize();
+    Ptr.activate();
+  }
+
+  return true;
+}
+
 // https://github.com/llvm/llvm-project/issues/102513
 #if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
 #pragma optimize("", off)
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index ae3d4a441a79..66d3e6d79e8b 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -189,7 +189,7 @@ bool CheckShift(InterpState &S, CodePtr OpPC, const LT &LHS, const RT &RHS,
 
   // C++11 [expr.shift]p1: Shift width must be less than the bit width of
   // the shifted type.
-  if (Bits > 1 && RHS >= RT::from(Bits, RHS.bitWidth())) {
+  if (Bits > 1 && RHS >= Bits) {
     const Expr *E = S.Current->getExpr(OpPC);
     const APSInt Val = RHS.toAPSInt();
     QualType Ty = E->getType();
@@ -370,6 +370,9 @@ bool AddSubMulHelper(InterpState &S, CodePtr OpPC, unsigned Bits, const T &LHS,
                      const T &RHS) {
   // Fast path - add the numbers with fixed width.
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!OpFW(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -408,6 +411,7 @@ bool Add(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
+
   return AddSubMulHelper<T, T::add, std::plus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -423,7 +427,7 @@ inline bool Addf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::add(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -434,6 +438,7 @@ bool Sub(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() + 1;
+
   return AddSubMulHelper<T, T::sub, std::minus>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -442,7 +447,7 @@ inline bool Subf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::sub(LHS, RHS, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -453,6 +458,7 @@ bool Mul(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
   const unsigned Bits = RHS.bitWidth() * 2;
+
   return AddSubMulHelper<T, T::mul, std::multiplies>(S, OpPC, Bits, LHS, RHS);
 }
 
@@ -461,8 +467,10 @@ inline bool Mulf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
   const Floating &LHS = S.Stk.pop<Floating>();
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+  Floating Result = S.allocFloat(LHS.getSemantics());
+
   auto Status = Floating::mul(LHS, RHS, getRoundingMode(FPO), &Result);
+
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -484,9 +492,14 @@ inline bool Mulc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexMul(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    Floating RA = S.allocFloat(A.getSemantics());
+    RA.copy(ResR);
+    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
     Result.atIndex(0).initialize();
-    Result.atIndex(1).deref<Floating>() = Floating(ResI);
+
+    Floating RI = S.allocFloat(A.getSemantics());
+    RI.copy(ResI);
+    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
     Result.atIndex(1).initialize();
     Result.initialize();
   } else {
@@ -539,10 +552,20 @@ inline bool Divc(InterpState &S, CodePtr OpPC) {
     HandleComplexComplexDiv(A, B, C, D, ResR, ResI);
 
     // Copy into the result.
-    Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    // Result.atIndex(0).deref<Floating>() = Floating(ResR);
+    // Result.atIndex(0).initialize();
+    // Result.atIndex(1).deref<Floating>() = Floating(ResI);
+    // Result.atIndex(1).initialize();
+
+    Floating RA = S.allocFloat(A.getSemantics());
+    RA.copy(ResR);
+    Result.atIndex(0).deref<Floating>() = RA; // Floating(ResR);
     Result.atIndex(0).initialize();
-    Result.atIndex(1).deref<Floating>() = Floating(ResI);
-    Result.atIndex(1).initialize();
+
+    Floating RI = S.allocFloat(A.getSemantics());
+    RI.copy(ResI);
+    Result.atIndex(1).deref<Floating>() = RI; // Floating(ResI);
+
     Result.initialize();
   } else {
     // Integer element type.
@@ -608,9 +631,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitAnd(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitAnd(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -625,9 +651,12 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool BitOr(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
-
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitOr(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -644,7 +673,11 @@ bool BitXor(InterpState &S, CodePtr OpPC) {
   const T &LHS = S.Stk.pop<T>();
 
   unsigned Bits = RHS.bitWidth();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Bits);
+
   if (!T::bitXor(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -659,12 +692,15 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Rem(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
+  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
-  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!T::rem(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -679,12 +715,15 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Div(InterpState &S, CodePtr OpPC) {
   const T &RHS = S.Stk.pop<T>();
   const T &LHS = S.Stk.pop<T>();
+  const unsigned Bits = RHS.bitWidth() * 2;
 
   if (!CheckDivRem(S, OpPC, LHS, RHS))
     return false;
 
-  const unsigned Bits = RHS.bitWidth() * 2;
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(LHS.bitWidth());
+
   if (!T::div(LHS, RHS, Bits, &Result)) {
     S.Stk.push<T>(Result);
     return true;
@@ -707,8 +746,10 @@ inline bool Divf(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
     return false;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  Floating Result;
+
+  Floating Result = S.allocFloat(LHS.getSemantics());
   auto Status = Floating::div(LHS, RHS, getRoundingMode(FPO), &Result);
+
   S.Stk.push<Floating>(Result);
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
 }
@@ -730,31 +771,44 @@ inline bool Inv(InterpState &S, CodePtr OpPC) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Neg(InterpState &S, CodePtr OpPC) {
   const T &Value = S.Stk.pop<T>();
-  T Result;
 
-  if (!T::neg(Value, &Result)) {
+  if constexpr (std::is_same_v<T, Floating>) {
+    T Result = S.allocFloat(Value.getSemantics());
+
+    if (!T::neg(Value, &Result)) {
+      S.Stk.push<T>(Result);
+      return true;
+    }
+    return false;
+  } else {
+    T Result;
+    if constexpr (needsAlloc<T>())
+      Result = S.allocAP<T>(Value.bitWidth());
+
+    if (!T::neg(Value, &Result)) {
+      S.Stk.push<T>(Result);
+      return true;
+    }
+
+    assert(isIntegralType(Name) &&
+           "don't expect other types to fail at constexpr negation");
     S.Stk.push<T>(Result);
-    return true;
-  }
 
-  assert(isIntegralType(Name) &&
-         "don't expect other types to fail at constexpr negation");
-  S.Stk.push<T>(Result);
+    APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
+    if (S.checkingForUndefinedBehavior()) {
+      const Expr *E = S.Current->getExpr(OpPC);
+      QualType Type = E->getType();
+      SmallString<32> Trunc;
+      NegatedValue.trunc(Result.bitWidth())
+          .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
+                    /*UpperCase=*/true, /*InsertSeparators=*/true);
+      S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
+          << Trunc << Type << E->getSourceRange();
+      return true;
+    }
 
-  APSInt NegatedValue = -Value.toAPSInt(Value.bitWidth() + 1);
-  if (S.checkingForUndefinedBehavior()) {
-    const Expr *E = S.Current->getExpr(OpPC);
-    QualType Type = E->getType();
-    SmallString<32> Trunc;
-    NegatedValue.trunc(Result.bitWidth())
-        .toString(Trunc, 10, Result.isSigned(), /*formatAsCLiteral=*/false,
-                  /*UpperCase=*/true, /*InsertSeparators=*/true);
-    S.report(E->getExprLoc(), diag::warn_integer_constant_overflow)
-        << Trunc << Type << E->getSourceRange();
-    return true;
+    return handleOverflow(S, OpPC, NegatedValue);
   }
-
-  return handleOverflow(S, OpPC, NegatedValue);
 }
 
 enum class PushVal : bool {
@@ -783,6 +837,8 @@ bool IncDecHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
 
   const T &Value = Ptr.deref<T>();
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Value.bitWidth());
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<T>(Value);
@@ -890,7 +946,6 @@ bool PreDec(InterpState &S, CodePtr OpPC, bool CanOverflow) {
   const Pointer &Ptr = S.Stk.peek<Pointer>();
   if (!CheckLoad(S, OpPC, Ptr, AK_Decrement))
     return false;
-
   return IncDecHelper<T, IncDecOp::Dec, PushVal::No>(S, OpPC, Ptr, CanOverflow);
 }
 
@@ -898,7 +953,7 @@ template <IncDecOp Op, PushVal DoPush>
 bool IncDecFloatHelper(InterpState &S, CodePtr OpPC, const Pointer &Ptr,
                        uint32_t FPOI) {
   Floating Value = Ptr.deref<Floating>();
-  Floating Result;
+  Floating Result = S.allocFloat(Value.getSemantics());
 
   if constexpr (DoPush == PushVal::Yes)
     S.Stk.push<Floating>(Value);
@@ -952,12 +1007,15 @@ inline bool DecfPop(InterpState &S, CodePtr OpPC, uint32_t FPOI) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Comp(InterpState &S, CodePtr OpPC) {
   const T &Val = S.Stk.pop<T>();
+
   T Result;
+  if constexpr (needsAlloc<T>())
+    Result = S.allocAP<T>(Val.bitWidth());
+
   if (!T::comp(Val, &Result)) {
     S.Stk.push<T>(Result);
     return true;
   }
-
   return false;
 }
 
@@ -1325,10 +1383,23 @@ bool Flip(InterpState &S, CodePtr OpPC) {
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool Const(InterpState &S, CodePtr OpPC, const T &Arg) {
+  if constexpr (needsAlloc<T>()) {
+    T Result = S.allocAP<T>(Arg.bitWidth());
+    Result.copy(Arg.toAPSInt());
+    S.Stk.push<T>(Result);
+    return true;
+  }
   S.Stk.push<T>(Arg);
   return true;
 }
 
+inline bool ConstFloat(InterpState &S, CodePtr OpPC, const Floating &F) {
+  Floating Result = S.allocFloat(F.getSemantics());
+  Result.copy(F.getAPFloat());
+  S.Stk.push<Floating>(Result);
+  return true;
+}
+
 //===----------------------------------------------------------------------===//
 // Get/Set Local/Param/Global/This
 //===----------------------------------------------------------------------===//
@@ -1483,7 +1554,24 @@ bool SetGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool InitGlobal(InterpState &S, CodePtr OpPC, uint32_t I) {
   const Pointer &P = S.P.getGlobal(I);
+
   P.deref<T>() = S.Stk.pop<T>();
+
+  if constexpr (std::is_same_v<T, Floating>) {
+    auto &Val = P.deref<Floating>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+
+  } else if constexpr (needsAlloc<T>()) {
+    auto &Val = P.deref<T>();
+    if (!Val.singleWord()) {
+      uint64_t *NewMemory = new (S.P) uint64_t[Val.numWords()];
+      Val.take(NewMemory);
+    }
+  }
+
   P.initialize();
   return true;
 }
@@ -1585,7 +1673,22 @@ bool InitBitField(InterpState &S, CodePtr OpPC, const Record::Field *F) {
   assert(F->isBitField());
   const T &Value = S.Stk.pop<T>();
   const Pointer &Field = S.Stk.peek<Pointer>().atField(F->Offset);
-  Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
+
+  if constexpr (needsAlloc<T>()) {
+    T Result = S.allocAP<T>(Value.bitWidth());
+    if (T::isSigned())
+      Result.copy(Value.toAPSInt()
+                      .trunc(F->Decl->getBitWidthValue())
+                      .sextOrTrunc(Value.bitWidth()));
+    else
+      Result.copy(Value.toAPSInt()
+                      .trunc(F->Decl->getBitWidthValue())
+                      .zextOrTrunc(Value.bitWidth()));
+
+    Field.deref<T>() = Result;
+  } else {
+    Field.deref<T>() = Value.truncate(F->Decl->getBitWidthValue());
+  }
   Field.activate();
   Field.initialize();
   return true;
@@ -1765,6 +1868,8 @@ inline bool FinishInit(InterpState &S, CodePtr OpPC) {
   return true;
 }
 
+bool FinishInitGlobal(InterpState &S, CodePtr OpPC);
+
 inline bool Dump(InterpState &S, CodePtr OpPC) {
   S.Stk.dump();
   return true;
@@ -2271,7 +2376,8 @@ template <PrimType TIn, PrimType TOut> bool Cast(InterpState &S, CodePtr OpPC) {
 inline bool CastFP(InterpState &S, CodePtr OpPC, const llvm::fltSemantics *Sem,
                    llvm::RoundingMode RM) {
   Floating F = S.Stk.pop<Floating>();
-  Floating Result = F.toSemantics(Sem, RM);
+  Floating Result = S.allocFloat(*Sem);
+  F.toSemantics(Sem, RM, &Result);
   S.Stk.push<Floating>(Result);
   return true;
 }
@@ -2295,15 +2401,25 @@ inline bool CastFixedPoint(InterpState &S, CodePtr OpPC, uint32_t FPS) {
 /// to know what bitwidth the result should be.
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<false>>(
-      IntegralAP<false>::from(S.Stk.pop<T>(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  // Copy data.
+  {
+    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
+    Result.copy(Source);
+  }
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
 template <PrimType Name, class T = typename PrimConv<Name>::T>
 bool CastAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<true>>(
-      IntegralAP<true>::from(S.Stk.pop<T>(), BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  // Copy data.
+  {
+    APInt Source = S.Stk.pop<T>().toAPSInt().extOrTrunc(BitWidth);
+    Result.copy(Source);
+  }
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2312,11 +2428,11 @@ bool CastIntegralFloating(InterpState &S, CodePtr OpPC,
                           const llvm::fltSemantics *Sem, uint32_t FPOI) {
   const T &From = S.Stk.pop<T>();
   APSInt FromAP = From.toAPSInt();
-  Floating Result;
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
+  Floating Result = S.allocFloat(*Sem);
   auto Status =
-      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), Result);
+      Floating::fromIntegral(FromAP, *Sem, getRoundingMode(FPO), &Result);
   S.Stk.push<Floating>(Result);
 
   return CheckFloatResult(S, OpPC, Result, Status, FPO);
@@ -2365,7 +2481,12 @@ static inline bool CastFloatingIntegralAP(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+
+  auto ResultAP = S.allocAP<IntegralAP<false>>(BitWidth);
+  ResultAP.copy(Result);
+
+  S.Stk.push<IntegralAP<false>>(ResultAP);
+
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2381,7 +2502,12 @@ static inline bool CastFloatingIntegralAPS(InterpState &S, CodePtr OpPC,
     return handleOverflow(S, OpPC, F.getAPFloat());
 
   FPOptions FPO = FPOptions::getFromOpaqueInt(FPOI);
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>(Result));
+
+  auto ResultAP = S.allocAP<IntegralAP<true>>(BitWidth);
+  ResultAP.copy(Result);
+
+  S.Stk.push<IntegralAP<true>>(ResultAP);
+
   return CheckFloatResult(S, OpPC, F, Status, FPO);
 }
 
@@ -2441,8 +2567,9 @@ static inline bool CastFloatingFixedPoint(InterpState &S, CodePtr OpPC,
 static inline bool CastFixedPointFloating(InterpState &S, CodePtr OpPC,
                                           const llvm::fltSemantics *Sem) {
   const auto &Fixed = S.Stk.pop<FixedPoint>();
-
-  S.Stk.push<Floating>(Fixed.toFloat(Sem));
+  Floating Result = S.allocFloat(*Sem);
+  Result.copy(Fixed.toFloat(Sem));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -2506,12 +2633,18 @@ bool Zero(InterpState &S, CodePtr OpPC) {
 }
 
 static inline bool ZeroIntAP(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<false>>(IntegralAP<false>::zero(BitWidth));
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  if (!Result.singleWord())
+    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
+  S.Stk.push<IntegralAP<false>>(Result);
   return true;
 }
 
 static inline bool ZeroIntAPS(InterpState &S, CodePtr OpPC, uint32_t BitWidth) {
-  S.Stk.push<IntegralAP<true>>(IntegralAP<true>::zero(BitWidth));
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  if (!Result.singleWord())
+    std::memset(Result.Memory, 0, Result.numWords() * sizeof(uint64_t));
+  S.Stk.push<IntegralAP<true>>(Result);
   return true;
 }
 
@@ -2578,7 +2711,9 @@ inline bool RVOPtr(InterpState &S, CodePtr OpPC) {
 //===----------------------------------------------------------------------===//
 
 template <class LT, class RT, ShiftDir Dir>
-inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
+inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
+                    LT *Result) {
+
   const unsigned Bits = LHS.bitWidth();
 
   // OpenCL 6.3j: shift values are effectively % word size of LHS.
@@ -2596,7 +2731,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
     RHS = -RHS;
     return DoShift<LT, RT,
                    Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
-        S, OpPC, LHS, RHS);
+        S, OpPC, LHS, RHS, Result);
   }
 
   if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
@@ -2644,6 +2779,7 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
       // Do the shift on potentially signed LT, then convert to unsigned type.
       LT A;
       LT::shiftRight(LHS, LT::from(RHS, Bits), Bits, &A);
+      // LT::shiftRight(LHS, LT(RHSTemp), Bits, &A);
       R = LT::AsUnsigned::from(A);
     }
   }
@@ -2652,6 +2788,48 @@ inline bool DoShift(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS) {
   return true;
 }
 
+/// A version of DoShift that works on IntegralAP.
+template <class LT, class RT, ShiftDir Dir>
+inline bool DoShiftAP(InterpState &S, CodePtr OpPC, LT &LHS, RT &RHS,
+                      LT *Result) {
+  const unsigned Bits = LHS.bitWidth();
+  const APSInt &LHSAP = LHS.toAPSInt();
+  APSInt RHSAP = RHS.toAPSInt();
+
+  // OpenCL 6.3j: shift values are effectively % word size of LHS.
+  if (S.getLangOpts().OpenCL)
+    RHSAP &= APSInt(llvm::APInt(RHSAP.getBitWidth(),
+                                static_cast<uint64_t>(LHSAP.getBitWidth() - 1)),
+                    RHSAP.isUnsigned());
+
+  if (RHS.isNegative()) {
+    // During constant-folding, a negative shift is an opposite shift. Such a
+    // shift is not a constant expression.
+    const SourceInfo &Loc = S.Current->getSource(OpPC);
+    S.CCEDiag(Loc, diag::note_constexpr_negative_shift) << RHS.toAPSInt();
+    if (!S.noteUndefinedBehavior())
+      return false;
+    RHS = -RHS;
+    return DoShiftAP<LT, RT,
+                     Dir == ShiftDir::Left ? ShiftDir::Right : ShiftDir::Left>(
+        S, OpPC, LHS, RHS, Result);
+  }
+
+  if (!CheckShift<Dir>(S, OpPC, LHS, RHS, Bits))
+    return false;
+
+  if constexpr (Dir == ShiftDir::Left) {
+    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
+    Result->copy(LHSAP << SA);
+  } else {
+    unsigned SA = (unsigned)RHSAP.getLimitedValue(LHS.bitWidth() - 1);
+    Result->copy(LHSAP >> SA);
+  }
+
+  S.Stk.push<LT>(*Result);
+  return true;
+}
+
 template <PrimType NameL, PrimType NameR>
 inline bool Shr(InterpState &S, CodePtr OpPC) {
   using LT = typename PrimConv<NameL>::T;
@@ -2659,7 +2837,13 @@ inline bool Shr(InterpState &S, CodePtr OpPC) {
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
 
-  return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS);
+  if constexpr (needsAlloc<LT>()) {
+    LT Result = S.allocAP<LT>(LHS.bitWidth());
+    return DoShiftAP<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
+  } else {
+    LT Result;
+    return DoShift<LT, RT, ShiftDir::Right>(S, OpPC, LHS, RHS, &Result);
+  }
 }
 
 template <PrimType NameL, PrimType NameR>
@@ -2668,8 +2852,13 @@ inline bool Shl(InterpState &S, CodePtr OpPC) {
   using RT = typename PrimConv<NameR>::T;
   auto RHS = S.Stk.pop<RT>();
   auto LHS = S.Stk.pop<LT>();
-
-  return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS);
+  if constexpr (needsAlloc<LT>()) {
+    LT Result = S.allocAP<LT>(LHS.bitWidth());
+    return DoShiftAP<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
+  } else {
+    LT Result;
+    return DoShift<LT, RT, ShiftDir::Left>(S, OpPC, LHS, RHS, &Result);
+  }
 }
 
 static inline bool ShiftFixedPoint(InterpState &S, CodePtr OpPC, bool Left) {
@@ -3252,7 +3441,15 @@ inline bool BitCastPrim(InterpState &S, CodePtr OpPC, bool TargetIsUCharOrByte,
 
     if constexpr (std::is_same_v<T, Floating>) {
       assert(Sem);
-      S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
+      Floating Result = S.allocFloat(*Sem);
+      Floating::bitcastFromMemory(Buff.data(), *Sem, &Result);
+      S.Stk.push<Floating>(Result);
+
+      // S.Stk.push<Floating>(T::bitcastFromMemory(Buff.data(), *Sem));
+    } else if constexpr (needsAlloc<T>()) {
+      T Result = S.allocAP<T>(ResultBitWidth);
+      T::bitcastFromMemory(Buff.data(), ResultBitWidth, &Result);
+      S.Stk.push<T>(Result);
     } else {
       assert(!Sem);
       S.Stk.push<T>(T::bitcastFromMemory(Buff.data(), ResultBitWidth));
@@ -3310,7 +3507,11 @@ template <typename T> inline T ReadArg(InterpState &S, CodePtr &OpPC) {
 }
 
 template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
-  Floating F = Floating::deserialize(*OpPC);
+  auto &Semantics =
+      llvm::APFloatBase::EnumToSemantics(Floating::deserializeSemantics(*OpPC));
+
+  auto F = S.allocFloat(Semantics);
+  Floating::deserialize(*OpPC, &F);
   OpPC += align(F.bytesToSerialize());
   return F;
 }
@@ -3318,17 +3519,25 @@ template <> inline Floating ReadArg<Floating>(InterpState &S, CodePtr &OpPC) {
 template <>
 inline IntegralAP<false> ReadArg<IntegralAP<false>>(InterpState &S,
                                                     CodePtr &OpPC) {
-  IntegralAP<false> I = IntegralAP<false>::deserialize(*OpPC);
-  OpPC += align(I.bytesToSerialize());
-  return I;
+  uint32_t BitWidth = IntegralAP<false>::deserializeSize(*OpPC);
+  auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+  assert(Result.bitWidth() == BitWidth);
+
+  IntegralAP<false>::deserialize(*OpPC, &Result);
+  OpPC += align(Result.bytesToSerialize());
+  return Result;
 }
 
 template <>
 inline IntegralAP<true> ReadArg<IntegralAP<true>>(InterpState &S,
                                                   CodePtr &OpPC) {
-  IntegralAP<true> I = IntegralAP<true>::deserialize(*OpPC);
-  OpPC += align(I.bytesToSerialize());
-  return I;
+  uint32_t BitWidth = IntegralAP<true>::deserializeSize(*OpPC);
+  auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+  assert(Result.bitWidth() == BitWidth);
+
+  IntegralAP<true>::deserialize(*OpPC, &Result);
+  OpPC += align(Result.bytesToSerialize());
+  return Result;
 }
 
 template <>
diff --git a/clang/lib/AST/ByteCode/InterpBuiltin.cpp b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
index d01e3d042a8b..5304bd77f2c0 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltin.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltin.cpp
@@ -57,6 +57,21 @@ static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
   assert(T);
 
   unsigned BitWidth = S.getASTContext().getTypeSize(QT);
+
+  if (T == PT_IntAPS) {
+    auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
+    Result.copy(Val);
+    S.Stk.push<IntegralAP<true>>(Result);
+    return;
+  }
+
+  if (T == PT_IntAP) {
+    auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
+    Result.copy(Val);
+    S.Stk.push<IntegralAP<false>>(Result);
+    return;
+  }
+
   if (QT->isSignedIntegerOrEnumerationType()) {
     int64_t V = Val.getSExtValue();
     INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
@@ -327,13 +342,13 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  Floating Result;
+  Floating Result = S.allocFloat(TargetSemantics);
   if (S.getASTContext().getTargetInfo().isNan2008()) {
     if (Signaling)
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
   } else {
     // Prior to IEEE 754-2008, architectures were allowed to choose whether
@@ -342,10 +357,10 @@ static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
     // 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
     // sNaN. This is now known as "legacy NaN" encoding.
     if (Signaling)
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
     else
-      Result = Floating(
+      Result.copy(
           llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
   }
 
@@ -360,7 +375,9 @@ static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
       S.getASTContext().getFloatTypeSemantics(
           Call->getDirectCallee()->getReturnType());
 
-  S.Stk.push<Floating>(Floating::getInf(TargetSemantics));
+  Floating Result = S.allocFloat(TargetSemantics);
+  Result.copy(APFloat::getInf(TargetSemantics));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -368,10 +385,12 @@ static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
                                      const InterpFrame *Frame) {
   const Floating &Arg2 = S.Stk.pop<Floating>();
   const Floating &Arg1 = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(Arg1.getSemantics());
 
   APFloat Copy = Arg1.getAPFloat();
   Copy.copySign(Arg2.getAPFloat());
-  S.Stk.push<Floating>(Floating(Copy));
+  Result.copy(Copy);
+  S.Stk.push<Floating>(Result);
 
   return true;
 }
@@ -380,11 +399,13 @@ static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    S.Stk.push<Floating>(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    S.Stk.push<Floating>(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -392,11 +413,13 @@ static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame, bool IsNumBuiltin) {
   const Floating &RHS = S.Stk.pop<Floating>();
   const Floating &LHS = S.Stk.pop<Floating>();
+  Floating Result = S.allocFloat(LHS.getSemantics());
 
   if (IsNumBuiltin)
-    S.Stk.push<Floating>(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
   else
-    S.Stk.push<Floating>(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
+    Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
@@ -571,8 +594,16 @@ static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
 static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
                                  const InterpFrame *Frame) {
   const Floating &Val = S.Stk.pop<Floating>();
+  APFloat F = Val.getAPFloat();
+  if (!F.isNegative()) {
+    S.Stk.push<Floating>(Val);
+    return true;
+  }
 
-  S.Stk.push<Floating>(Floating::abs(Val));
+  Floating Result = S.allocFloat(Val.getSemantics());
+  F.changeSign();
+  Result.copy(F);
+  S.Stk.push<Floating>(Result);
   return true;
 }
 
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 239b3104e89f..2569cac018b3 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -402,7 +402,9 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
           if (llvm::sys::IsBigEndianHost)
             swapBytes(M.get(), NumBits.roundToBytes());
 
-          P.deref<Floating>() = Floating::bitcastFromMemory(M.get(), Semantics);
+          Floating R = S.allocFloat(Semantics);
+          Floating::bitcastFromMemory(M.get(), Semantics, &R);
+          P.deref<Floating>() = R;
           P.initialize();
           return true;
         }
diff --git a/clang/lib/AST/ByteCode/InterpState.h b/clang/lib/AST/ByteCode/InterpState.h
index e8dc6f0483d6..08765561985e 100644
--- a/clang/lib/AST/ByteCode/InterpState.h
+++ b/clang/lib/AST/ByteCode/InterpState.h
@@ -15,6 +15,7 @@
 
 #include "Context.h"
 #include "DynamicAllocator.h"
+#include "Floating.h"
 #include "Function.h"
 #include "InterpFrame.h"
 #include "InterpStack.h"
@@ -126,6 +127,33 @@ public:
 
   StdAllocatorCaller getStdAllocatorCaller(StringRef Name) const;
 
+  void *allocate(size_t Size, unsigned Align = 8) const {
+    return Allocator.Allocate(Size, Align);
+  }
+  template <typename T> T *allocate(size_t Num = 1) const {
+    return static_cast<T *>(allocate(Num * sizeof(T), alignof(T)));
+  }
+
+  template <typename T> T allocAP(unsigned BitWidth) {
+    unsigned NumWords = APInt::getNumWords(BitWidth);
+    if (NumWords == 1)
+      return T(BitWidth);
+    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
+    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
+    return T(Mem, BitWidth);
+  }
+
+  Floating allocFloat(const llvm::fltSemantics &Sem) {
+    if (Floating::singleWord(Sem))
+      return Floating(llvm::APFloatBase::SemanticsToEnum(Sem));
+
+    unsigned NumWords =
+        APInt::getNumWords(llvm::APFloatBase::getSizeInBits(Sem));
+    uint64_t *Mem = (uint64_t *)this->allocate(NumWords * sizeof(uint64_t));
+    // std::memset(Mem, 0, NumWords * sizeof(uint64_t)); // Debug
+    return Floating(Mem, llvm::APFloatBase::SemanticsToEnum(Sem));
+  }
+
 private:
   friend class EvaluationResult;
   friend class InterpStateCCOverride;
@@ -161,6 +189,8 @@ public:
   llvm::SmallVector<
       std::pair<const Expr *, const LifetimeExtendedTemporaryDecl *>>
       SeenGlobalTemporaries;
+
+  mutable llvm::BumpPtrAllocator Allocator;
 };
 
 class InterpStateCCOverride final {
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index c76ac5f8ae86..57e01f7bd9da 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -48,6 +48,7 @@ def ArgUint64 : ArgType { let Name = "uint64_t"; }
 def ArgIntAP : ArgType { let Name = "IntegralAP<false>"; let AsRef = true; }
 def ArgIntAPS : ArgType { let Name = "IntegralAP<true>"; let AsRef = true; }
 def ArgFloat : ArgType { let Name = "Floating"; let AsRef = true; }
+
 def ArgBool : ArgType { let Name = "bool"; }
 def ArgFixedPoint : ArgType { let Name = "FixedPoint"; let AsRef = true; }
 
@@ -88,6 +89,9 @@ def IntegerAndFixedTypeClass : TypeClass {
                Uint32, Sint64, Uint64, IntAP, IntAPS, FixedPoint];
 }
 
+def IntegralTypeClass : TypeClass {
+  let Types = !listconcat(IntegerTypeClass.Types, [Bool]);
+}
 def FixedSizeIntegralTypeClass : TypeClass {
   let Types = [Sint8, Uint8, Sint16, Uint16, Sint32,
                Uint32, Sint64, Uint64, Bool];
@@ -265,12 +269,13 @@ def ConstSint32 : ConstOpcode<Sint32, ArgSint32>;
 def ConstUint32 : ConstOpcode<Uint32, ArgUint32>;
 def ConstSint64 : ConstOpcode<Sint64, ArgSint64>;
 def ConstUint64 : ConstOpcode<Uint64, ArgUint64>;
-def ConstFloat : ConstOpcode<Float, ArgFloat>;
-def constIntAP : ConstOpcode<IntAP, ArgIntAP>;
-def constIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
+def ConstIntAP : ConstOpcode<IntAP, ArgIntAP>;
+def ConstIntAPS : ConstOpcode<IntAPS, ArgIntAPS>;
 def ConstBool : ConstOpcode<Bool, ArgBool>;
 def ConstFixedPoint : ConstOpcode<FixedPoint, ArgFixedPoint>;
 
+def ConstFloat : Opcode { let Args = [ArgFloat]; }
+
 // [] -> [Integer]
 def Zero : Opcode {
   let Types = [FixedSizeIntegralTypeClass];
@@ -328,6 +333,7 @@ def GetMemberPtrBasePop : Opcode {
 
 def FinishInitPop : Opcode;
 def FinishInit    : Opcode;
+def FinishInitGlobal : Opcode;
 
 def GetPtrDerivedPop : Opcode { let Args = [ArgUint32, ArgBool, ArgTypePtr]; }
 
@@ -389,7 +395,7 @@ class AccessOpcode : Opcode {
 }
 
 class BitFieldOpcode : Opcode {
-  let Types = [AluTypeClass];
+  let Types = [IntegralTypeClass];
   let Args = [ArgRecordField];
   let HasGroup = 1;
 }
diff --git a/clang/lib/AST/ByteCode/PrimType.h b/clang/lib/AST/ByteCode/PrimType.h
index 6152fbfbe3a7..a156cccbb3c1 100644
--- a/clang/lib/AST/ByteCode/PrimType.h
+++ b/clang/lib/AST/ByteCode/PrimType.h
@@ -76,6 +76,13 @@ inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
 }
 
 constexpr bool isIntegralType(PrimType T) { return T <= PT_FixedPoint; }
+template <typename T> constexpr bool needsAlloc() {
+  return std::is_same_v<T, IntegralAP<false>> ||
+         std::is_same_v<T, IntegralAP<true>> || std::is_same_v<T, Floating>;
+}
+constexpr bool needsAlloc(PrimType T) {
+  return T == PT_IntAP || T == PT_IntAPS || T == PT_Float;
+}
 
 /// Mapping from primitive types to their representation.
 template <PrimType T> struct PrimConv;
@@ -209,6 +216,16 @@ static inline bool aligned(const void *P) {
     }                                                                          \
   } while (0)
 
+#define TYPE_SWITCH_ALLOC(Expr, B)                                             \
+  do {                                                                         \
+    switch (Expr) {                                                            \
+      TYPE_SWITCH_CASE(PT_Float, B)                                            \
+      TYPE_SWITCH_CASE(PT_IntAP, B)                                            \
+      TYPE_SWITCH_CASE(PT_IntAPS, B)                                           \
+    default:;                                                                  \
+    }                                                                          \
+  } while (0)
+
 #define COMPOSITE_TYPE_SWITCH(Expr, B, D)                                      \
   do {                                                                         \
     switch (Expr) {                                                            \
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index 23ba1bbd193b..5d9c42244749 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -132,6 +132,14 @@ public:
                                bool IsMutable = false, bool IsVolatile = false,
                                const Expr *Init = nullptr);
 
+  void *Allocate(size_t Size, unsigned Align = 8) const {
+    return Allocator.Allocate(Size, Align);
+  }
+  template <typename T> T *Allocate(size_t Num = 1) const {
+    return static_cast<T *>(Allocate(Num * sizeof(T), alignof(T)));
+  }
+  void Deallocate(void *Ptr) const {}
+
   /// Context to manage declaration lifetimes.
   class DeclScope {
   public:
@@ -204,7 +212,7 @@ private:
   };
 
   /// Allocator for globals.
-  PoolAllocTy Allocator;
+  mutable PoolAllocTy Allocator;
 
   /// Global objects.
   std::vector<Global *> Globals;
@@ -238,4 +246,18 @@ public:
 } // namespace interp
 } // namespace clang
 
+inline void *operator new(size_t Bytes, const clang::interp::Program &C,
+                          size_t Alignment = 8) {
+  return C.Allocate(Bytes, Alignment);
+}
+
+inline void operator delete(void *Ptr, const clang::interp::Program &C,
+                            size_t) {
+  C.Deallocate(Ptr);
+}
+inline void *operator new[](size_t Bytes, const clang::interp::Program &C,
+                            size_t Alignment = 8) {
+  return C.Allocate(Bytes, Alignment);
+}
+
 #endif
diff --git a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
index 710612bef8fd..1013a771d13b 100644
--- a/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
+++ b/clang/test/AST/ByteCode/builtin-bit-cast-long-double.cpp
@@ -21,6 +21,9 @@ template <class To, class From>
 constexpr To bit_cast(const From &from) {
   static_assert(sizeof(To) == sizeof(From));
   return __builtin_bit_cast(To, from);
+#if __x86_64
+  // both-note@-2 {{indeterminate value can only initialize an object of type}}
+#endif
 }
 
 template <class Intermediate, class Init>
@@ -38,11 +41,8 @@ constexpr Init round_trip(const Init &init) {
 
 namespace test_long_double {
 #if __x86_64
-/// FIXME: We could enable this, but since it aborts, it causes the usual mempory leak.
-#if 0
-constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // expected-error{{must be initialized by a constant expression}}\
-                                                                                 // expected-note{{in call}}
-#endif
+constexpr __int128_t test_cast_to_int128 = bit_cast<__int128_t>((long double)0); // both-error{{must be initialized by a constant expression}}\
+                                                                                 // both-note{{in call}}
 constexpr long double ld = 3.1425926539;
 
 struct bytes {
diff --git a/clang/test/AST/ByteCode/builtin-functions.cpp b/clang/test/AST/ByteCode/builtin-functions.cpp
index 21dca15a4577..174c1ffa79a4 100644
--- a/clang/test/AST/ByteCode/builtin-functions.cpp
+++ b/clang/test/AST/ByteCode/builtin-functions.cpp
@@ -208,7 +208,7 @@ namespace nan {
 
   constexpr double NaN3 = __builtin_nan("foo"); // both-error {{must be initialized by a constant expression}}
   constexpr float NaN4 = __builtin_nanf("");
-  //constexpr long double NaN5 = __builtin_nanf128("");
+  constexpr long double NaN5 = __builtin_nanf128("");
 
   /// FIXME: This should be accepted by the current interpreter as well.
   constexpr char f[] = {'0', 'x', 'A', 'E', '\0'};
@@ -655,8 +655,6 @@ void test_noexcept(int *i) {
 } // end namespace test_launder
 
 
-/// FIXME: The commented out tests here use a IntAP value and fail.
-/// This currently means we will leak the IntAP value since nothing cleans it up.
 namespace clz {
   char clz1[__builtin_clz(1) == BITSIZE(int) - 1 ? 1 : -1];
   char clz2[__builtin_clz(7) == BITSIZE(int) - 3 ? 1 : -1];
@@ -709,7 +707,7 @@ namespace clz {
   char clz48[__builtin_clzg(1ULL << (BITSIZE(long long) - 1)) == 0 ? 1 : -1];
   char clz49[__builtin_clzg(1ULL << (BITSIZE(long long) - 1), 42) == 0 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  // int clz50 = __builtin_clzg((unsigned __int128)0);
+  int clz50 = __builtin_clzg((unsigned __int128)0);
   char clz51[__builtin_clzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char clz52[__builtin_clzg((unsigned __int128)0x1) == BITSIZE(__int128) - 1 ? 1 : -1];
   char clz53[__builtin_clzg((unsigned __int128)0x1, 42) == BITSIZE(__int128) - 1 ? 1 : -1];
@@ -717,7 +715,7 @@ namespace clz {
   char clz55[__builtin_clzg((unsigned __int128)0xf, 42) == BITSIZE(__int128) - 4 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  // int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
+  int clz58 = __builtin_clzg((unsigned _BitInt(128))0);
   char clz59[__builtin_clzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char clz60[__builtin_clzg((unsigned _BitInt(128))0x1) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
   char clz61[__builtin_clzg((unsigned _BitInt(128))0x1, 42) == BITSIZE(_BitInt(128)) - 1 ? 1 : -1];
@@ -775,7 +773,7 @@ namespace ctz {
   char ctz46[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1)) == BITSIZE(long long) - 1 ? 1 : -1];
   char ctz47[__builtin_ctzg(1ULL << (BITSIZE(long long) - 1), 42) == BITSIZE(long long) - 1 ? 1 : -1];
 #ifdef __SIZEOF_INT128__
-  // int ctz48 = __builtin_ctzg((unsigned __int128)0);
+  int ctz48 = __builtin_ctzg((unsigned __int128)0);
   char ctz49[__builtin_ctzg((unsigned __int128)0, 42) == 42 ? 1 : -1];
   char ctz50[__builtin_ctzg((unsigned __int128)0x1) == 0 ? 1 : -1];
   char ctz51[__builtin_ctzg((unsigned __int128)0x1, 42) == 0 ? 1 : -1];
@@ -785,7 +783,7 @@ namespace ctz {
   char ctz55[__builtin_ctzg((unsigned __int128)1 << (BITSIZE(__int128) - 1), 42) == BITSIZE(__int128) - 1 ? 1 : -1];
 #endif
 #ifndef __AVR__
-  // int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
+  int ctz56 = __builtin_ctzg((unsigned _BitInt(128))0);
   char ctz57[__builtin_ctzg((unsigned _BitInt(128))0, 42) == 42 ? 1 : -1];
   char ctz58[__builtin_ctzg((unsigned _BitInt(128))0x1) == 0 ? 1 : -1];
   char ctz59[__builtin_ctzg((unsigned _BitInt(128))0x1, 42) == 0 ? 1 : -1];
-- 
cgit v1.2.3


From 770b16cd4939054e30f5bba40765cfe1f5895f74 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 20 Jun 2025 09:07:08 -0700
Subject: [BOLT][test] Update X86/perf2bolt-spe.test (#145061)

Address NFC mismatches caused by running perf2bolt from under the
wrapper script:
https://lab.llvm.org/buildbot/#/builders/92/builds/20938
> <stdin>:2:64: note: possible intended match here
>
/home/worker/bolt-worker2/bolt-x86_64-ubuntu-nfc/build/bin/llvm-bolt.old:
-spe is available only on AArch64.

Test Plan:
ninja check-bolt
---
 bolt/test/perf2bolt/X86/perf2bolt-spe.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bolt/test/perf2bolt/X86/perf2bolt-spe.test b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
index 8eed2c859509..101bd3789a18 100644
--- a/bolt/test/perf2bolt/X86/perf2bolt-spe.test
+++ b/bolt/test/perf2bolt/X86/perf2bolt-spe.test
@@ -6,4 +6,4 @@ RUN: %clang %cflags %p/../../Inputs/asm_foo.s %p/../../Inputs/asm_main.c -o %t.e
 RUN: touch %t.empty.perf.data
 RUN: not perf2bolt -p %t.empty.perf.data -o %t.perf.boltdata --spe --pa %t.exe 2>&1 | FileCheck %s
 
-CHECK: perf2bolt: -spe is available only on AArch64.
+CHECK: perf2bolt{{.*}} -spe is available only on AArch64.
-- 
cgit v1.2.3


From 72de0e45846bab293a0b07e5aeda65a5e6590a87 Mon Sep 17 00:00:00 2001
From: Chenguang Wang <w3cing@gmail.com>
Date: Fri, 20 Jun 2025 09:07:35 -0700
Subject: [TableGen][Docs] Fix empty list syntax in TableGen doc. (#145041)

`[]<list<int>>` actually produces `list<list<int>>`.
---
 llvm/docs/TableGen/ProgRef.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/TableGen/ProgRef.rst b/llvm/docs/TableGen/ProgRef.rst
index fe0564d2990d..7b30698ce4fd 100644
--- a/llvm/docs/TableGen/ProgRef.rst
+++ b/llvm/docs/TableGen/ProgRef.rst
@@ -1915,7 +1915,7 @@ and non-0 as true.
     ``list<int>``. *start* is ``0`` and *step* is ``1`` by default. *step* can
     be negative and cannot be 0. If *start* ``<`` *end* and *step* is negative,
     or *start* ``>`` *end* and *step* is positive, the result is an empty list
-    ``[]<list<int>>``.
+    ``[]<int>``.
 
     For example:
 
-- 
cgit v1.2.3


From 6ba1955ba22c0bea4d2fb76c6dfcad0f5354be72 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 20 Jun 2025 10:45:38 -0500
Subject: [flang][OpenMP] Fix ignore-target-data.f90 test

Allow the function definition line to match with and without attrbute
set number.

This fixes build break after PR144534:
https://lab.llvm.org/buildbot/#/builders/157/builds/31331

Also move the test to the OpenMP subdirectory where it should have
been from the beginning.
---
 flang/test/Lower/OpenMP/ignore-target-data.f90 | 30 ++++++++++++++++++++++++++
 flang/test/Lower/ignore-target-data.f90        | 30 --------------------------
 2 files changed, 30 insertions(+), 30 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/ignore-target-data.f90
 delete mode 100644 flang/test/Lower/ignore-target-data.f90

diff --git a/flang/test/Lower/OpenMP/ignore-target-data.f90 b/flang/test/Lower/OpenMP/ignore-target-data.f90
new file mode 100644
index 000000000000..f1a150d5dfab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/ignore-target-data.f90
@@ -0,0 +1,30 @@
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=NORT
+!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=LLVM
+
+!Make sure that there are no calls to the mapper.
+!NORT-NOT: call{{.*}}__tgt_target_data_begin_mapper
+!NORT-NOT: call{{.*}}__tgt_target_data_end_mapper
+
+!Make sure we generate the body
+!LLVM: define internal void @_QFPf(ptr %[[A0:[0-9]+]], ptr %[[A1:[0-9]+]])
+!LLVM:   %[[V0:[0-9]+]] = load i32, ptr %[[A0]], align 4
+!LLVM:   %[[V1:[0-9]+]] = load i32, ptr %[[A1]], align 4
+!LLVM:   %[[V2:[0-9]+]] = add i32 %[[V0]], %[[V1]]
+!LLVM:   store i32 %[[V2]], ptr %[[A0]], align 4
+!LLVM:   ret void
+!LLVM: }
+
+
+program test
+
+call f(1, 2)
+
+contains
+
+subroutine f(x, y)
+  integer :: x, y
+  !$omp target data map(tofrom: x, y)
+  x = x + y
+  !$omp end target data
+end subroutine
+end
diff --git a/flang/test/Lower/ignore-target-data.f90 b/flang/test/Lower/ignore-target-data.f90
deleted file mode 100644
index 08947c137090..000000000000
--- a/flang/test/Lower/ignore-target-data.f90
+++ /dev/null
@@ -1,30 +0,0 @@
-!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=NORT
-!RUN: %flang_fc1 -emit-llvm -fopenmp %s -o - | FileCheck %s --check-prefix=LLVM
-
-!Make sure that there are no calls to the mapper.
-!NORT-NOT: call{{.*}}__tgt_target_data_begin_mapper
-!NORT-NOT: call{{.*}}__tgt_target_data_end_mapper
-
-!Make sure we generate the body
-!LLVM: define internal void @_QFPf(ptr %[[A0:[0-9]+]], ptr %[[A1:[0-9]+]]) {
-!LLVM:   %[[V0:[0-9]+]] = load i32, ptr %[[A0]], align 4
-!LLVM:   %[[V1:[0-9]+]] = load i32, ptr %[[A1]], align 4
-!LLVM:   %[[V2:[0-9]+]] = add i32 %[[V0]], %[[V1]]
-!LLVM:   store i32 %[[V2]], ptr %[[A0]], align 4
-!LLVM:   ret void
-!LLVM: }
-
-
-program test
-
-call f(1, 2)
-
-contains
-
-subroutine f(x, y)
-  integer :: x, y
-  !$omp target data map(tofrom: x, y)
-  x = x + y
-  !$omp end target data
-end subroutine
-end
-- 
cgit v1.2.3


From bfef8732be1b7b3a7ba7b3ccd93d043fd044293e Mon Sep 17 00:00:00 2001
From: Justin King <jcking@google.com>
Date: Fri, 20 Jun 2025 09:16:40 -0700
Subject: msan: Support free_sized and free_aligned_sized from C23 (#144529)

Adds support to MSan for `free_sized` and `free_aligned_sized` from C23.

Other sanitizers will be handled with their own separate PRs.

For https://github.com/llvm/llvm-project/issues/144435

Signed-off-by: Justin King <jcking@google.com>
---
 compiler-rt/lib/msan/msan_interceptors.cpp         | 32 ++++++++++++++++++++++
 .../TestCases/Linux/free_aligned_sized.c           |  2 +-
 .../sanitizer_common/TestCases/Linux/free_sized.c  |  2 +-
 3 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index f94d3cb79aa0..bc28434d1d49 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -34,6 +34,7 @@
 #include "sanitizer_common/sanitizer_glibc_version.h"
 #include "sanitizer_common/sanitizer_libc.h"
 #include "sanitizer_common/sanitizer_linux.h"
+#include "sanitizer_common/sanitizer_platform_interceptors.h"
 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
@@ -215,6 +216,35 @@ INTERCEPTOR(void, free, void *ptr) {
   MsanDeallocate(&stack, ptr);
 }
 
+#if SANITIZER_INTERCEPT_FREE_SIZED
+INTERCEPTOR(void, free_sized, void *ptr, uptr size) {
+  if (UNLIKELY(!ptr))
+    return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_MALLOC_STACK_TRACE;
+  MsanDeallocate(&stack, ptr);
+}
+#  define MSAN_MAYBE_INTERCEPT_FREE_SIZED INTERCEPT_FUNCTION(free_sized)
+#else
+#  define MSAN_MAYBE_INTERCEPT_FREE_SIZED
+#endif
+
+#if SANITIZER_INTERCEPT_FREE_ALIGNED_SIZED
+INTERCEPTOR(void, free_aligned_sized, void *ptr, uptr alignment, uptr size) {
+  if (UNLIKELY(!ptr))
+    return;
+  if (DlsymAlloc::PointerIsMine(ptr))
+    return DlsymAlloc::Free(ptr);
+  GET_MALLOC_STACK_TRACE;
+  MsanDeallocate(&stack, ptr);
+}
+#  define MSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED \
+    INTERCEPT_FUNCTION(free_aligned_sized)
+#else
+#  define MSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED
+#endif
+
 #if !SANITIZER_FREEBSD && !SANITIZER_NETBSD
 INTERCEPTOR(void, cfree, void *ptr) {
   if (UNLIKELY(!ptr))
@@ -1775,6 +1805,8 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(realloc);
   INTERCEPT_FUNCTION(reallocarray);
   INTERCEPT_FUNCTION(free);
+  MSAN_MAYBE_INTERCEPT_FREE_SIZED;
+  MSAN_MAYBE_INTERCEPT_FREE_ALIGNED_SIZED;
   MSAN_MAYBE_INTERCEPT_CFREE;
   MSAN_MAYBE_INTERCEPT_MALLOC_USABLE_SIZE;
   MSAN_MAYBE_INTERCEPT_MALLINFO;
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
index e9cb6f20c5ea..7710c6236819 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_aligned_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
index 8cdf3216e528..9eac562fecb0 100644
--- a/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
+++ b/compiler-rt/test/sanitizer_common/TestCases/Linux/free_sized.c
@@ -1,5 +1,5 @@
 // RUN: %clang -std=c23 -O0 %s -o %t && %run %t
-// UNSUPPORTED: asan, hwasan, rtsan, tsan, msan, ubsan
+// UNSUPPORTED: asan, hwasan, rtsan, tsan, ubsan
 
 #include <stddef.h>
 #include <stdlib.h>
-- 
cgit v1.2.3


From 877511920dcf36463e06746d626e8876583a6abd Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Fri, 20 Jun 2025 17:20:58 +0100
Subject: Revert "[lldb][DWARF] Remove object_pointer from
 ParsedDWARFAttributes" (#145065)

Reverts llvm/llvm-project#144880

Caused `TestObjCIvarsInBlocks.py` to fail on macOS CI.
---
 .../SymbolFile/DWARF/DWARFASTParserClang.cpp       | 28 +++++++++++++++-------
 .../Plugins/SymbolFile/DWARF/DWARFASTParserClang.h |  7 +++---
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index 3bec89cdf746..4f79c8aa3f81 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -445,6 +445,15 @@ ParsedDWARFTypeAttributes::ParsedDWARFTypeAttributes(const DWARFDIE &die) {
       name.SetCString(form_value.AsCString());
       break;
 
+    case DW_AT_object_pointer:
+      // GetAttributes follows DW_AT_specification.
+      // DW_TAG_subprogram definitions and declarations may both
+      // have a DW_AT_object_pointer. Don't overwrite the one
+      // we parsed for the definition with the one from the declaration.
+      if (!object_pointer.IsValid())
+        object_pointer = form_value.Reference();
+      break;
+
     case DW_AT_signature:
       signature = form_value;
       break;
@@ -1107,7 +1116,7 @@ bool DWARFASTParserClang::ParseObjCMethod(
 std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
     const DWARFDIE &die, CompilerType clang_type,
     const ParsedDWARFTypeAttributes &attrs, const DWARFDIE &decl_ctx_die,
-    const DWARFDIE &object_parameter, bool &ignore_containing_context) {
+    bool is_static, bool &ignore_containing_context) {
   Log *log = GetLog(DWARFLog::TypeCompletion | DWARFLog::Lookups);
   SymbolFileDWARF *dwarf = die.GetDWARF();
   assert(dwarf);
@@ -1191,9 +1200,6 @@ std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
       TypeSystemClang::GetDeclContextForType(class_opaque_type), die,
       attrs.name.GetCString());
 
-  // In DWARF, a C++ method is static if it has no object parameter child.
-  const bool is_static = !object_parameter.IsValid();
-
   // We have a C++ member function with no children (this pointer!) and clang
   // will get mad if we try and make a function that isn't well formed in the
   // DWARF, so we will just skip it...
@@ -1219,7 +1225,9 @@ std::pair<bool, TypeSP> DWARFASTParserClang::ParseCXXMethod(
     ClangASTMetadata metadata;
     metadata.SetUserID(die.GetID());
 
-    if (char const *object_pointer_name = object_parameter.GetName()) {
+    char const *object_pointer_name =
+        attrs.object_pointer ? attrs.object_pointer.GetName() : nullptr;
+    if (object_pointer_name) {
       metadata.SetObjectPtrName(object_pointer_name);
       LLDB_LOGF(log, "Setting object pointer name: %s on method object %p.\n",
                 object_pointer_name, static_cast<void *>(cxx_method_decl));
@@ -1315,9 +1323,11 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
         type_handled =
             ParseObjCMethod(*objc_method, die, clang_type, attrs, is_variadic);
       } else if (is_cxx_method) {
+        // In DWARF, a C++ method is static if it has no object parameter child.
+        const bool is_static = !object_parameter.IsValid();
         auto [handled, type_sp] =
-            ParseCXXMethod(die, clang_type, attrs, decl_ctx_die,
-                           object_parameter, ignore_containing_context);
+            ParseCXXMethod(die, clang_type, attrs, decl_ctx_die, is_static,
+                           ignore_containing_context);
         if (type_sp)
           return type_sp;
 
@@ -1412,7 +1422,9 @@ DWARFASTParserClang::ParseSubroutine(const DWARFDIE &die,
           ClangASTMetadata metadata;
           metadata.SetUserID(die.GetID());
 
-          if (char const *object_pointer_name = object_parameter.GetName()) {
+          char const *object_pointer_name =
+              attrs.object_pointer ? attrs.object_pointer.GetName() : nullptr;
+          if (object_pointer_name) {
             metadata.SetObjectPtrName(object_pointer_name);
             LLDB_LOGF(log,
                       "Setting object pointer name: %s on function "
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
index a90f55bcff94..111604ce4068 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.h
@@ -470,8 +470,7 @@ private:
   /// \param[in] decl_ctx_die The DIE representing the DeclContext of the C++
   ///                         method being parsed.
   ///
-  /// \param[in] object_parameter The DIE of this subprogram's object parameter.
-  ///                             May be an invalid DIE for C++ static methods.
+  /// \param[in] is_static Is true iff we're parsing a static method.
   ///
   /// \param[out] ignore_containing_context Will get set to true if the caller
   ///             should treat this C++ method as-if it was not a C++ method.
@@ -486,8 +485,7 @@ private:
                  lldb_private::CompilerType clang_type,
                  const ParsedDWARFTypeAttributes &attrs,
                  const lldb_private::plugin::dwarf::DWARFDIE &decl_ctx_die,
-                 const lldb_private::plugin::dwarf::DWARFDIE &object_parameter,
-                 bool &ignore_containing_context);
+                 bool is_static, bool &ignore_containing_context);
 
   lldb::TypeSP ParseArrayType(const lldb_private::plugin::dwarf::DWARFDIE &die,
                               const ParsedDWARFTypeAttributes &attrs);
@@ -557,6 +555,7 @@ struct ParsedDWARFTypeAttributes {
   const char *mangled_name = nullptr;
   lldb_private::ConstString name;
   lldb_private::Declaration decl;
+  lldb_private::plugin::dwarf::DWARFDIE object_pointer;
   lldb_private::plugin::dwarf::DWARFFormValue abstract_origin;
   lldb_private::plugin::dwarf::DWARFFormValue containing_type;
   lldb_private::plugin::dwarf::DWARFFormValue signature;
-- 
cgit v1.2.3


From 940ff110d78ba1eea7e0004275e098024cfd87c4 Mon Sep 17 00:00:00 2001
From: Jameson Nash <vtjnash@gmail.com>
Date: Fri, 20 Jun 2025 12:22:59 -0400
Subject: [InstCombine] fix hwasan mistake in "remove dead loads" (#145057)

Detected by CI after #143958.
---
 llvm/lib/Transforms/InstCombine/InstructionCombining.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index bcc73090277a..ce4202926135 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -3490,6 +3490,7 @@ Instruction *InstCombinerImpl::visitAllocSite(Instruction &MI) {
           replaceInstUsesWith(*I, Result);
           eraseInstFromFunction(*I);
           Users[i] = nullptr; // Skip examining in the next loop.
+          continue;
         }
         if (auto *MTI = dyn_cast<MemTransferInst>(I)) {
           if (KnowInitZero && isRefSet(*Removable)) {
-- 
cgit v1.2.3


From 379a609dadc1733c2b62d2bf3bab6e8032236836 Mon Sep 17 00:00:00 2001
From: Muzammil <55665739+Muzammiluddin-Syed-ECE@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:27:35 -0400
Subject: [mlir][arith][transforms] Adds f4E2M1FN support to truncf and extf
 (#144157)

See work detail: https://github.com/iree-org/iree/issues/20920

Add support for f4E2M1FN in `arith.truncf` and `arith.extf` ops though a software emulation

---------

Signed-off-by: Muzammiluddin Syed <muzasyed@amd.com>
---
 .../include/mlir/Dialect/Arith/Transforms/Passes.h |   3 +
 .../mlir/Dialect/Arith/Transforms/Passes.td        |   2 +
 mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp    | 258 ++++++++++++++++++++-
 mlir/test/Dialect/Arith/expand-ops.mlir            |  42 +++-
 .../Arith/CPU/test-arith-expand-truncf-extf.mlir   |  73 ++++++
 5 files changed, 370 insertions(+), 8 deletions(-)
 create mode 100644 mlir/test/Integration/Dialect/Arith/CPU/test-arith-expand-truncf-extf.mlir

diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
index e0a4567d6f40..b03cf2db7804 100644
--- a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.h
@@ -59,6 +59,9 @@ void populateCeilFloorDivExpandOpsPatterns(RewritePatternSet &patterns);
 /// Add patterns to expand Arith bf16 patterns to lower level bitcasts/shifts.
 void populateExpandBFloat16Patterns(RewritePatternSet &patterns);
 
+/// Add patterns to expand Arith f4e2m1 patterns to lower level bitcasts/shifts.
+void populateExpandF4E2M1Patterns(RewritePatternSet &patterns);
+
 /// Add patterns to expand Arith f8e8m0 patterns to lower level bitcasts/shifts.
 void populateExpandF8E8M0Patterns(RewritePatternSet &patterns);
 
diff --git a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
index e14b2aeee1c6..c7370b83fdb6 100644
--- a/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/Arith/Transforms/Passes.td
@@ -19,6 +19,8 @@ def ArithExpandOpsPass : Pass<"arith-expand"> {
               "Enable the BF16 expansion patterns">,
        Option<"includeF8E8M0", "include-f8e8m0", "bool", /*default=*/"false",
               "Enable the F8E8M0 expansion patterns">,
+       Option<"includeF4E2M1", "include-f4e2m1", "bool", /*default=*/"false",
+              "Enable the F4E2M1 expansion patterns">,
   ];
 }
 
diff --git a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
index 534aff9562b7..dfa01844737c 100644
--- a/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
+++ b/mlir/lib/Dialect/Arith/Transforms/ExpandOps.cpp
@@ -11,8 +11,11 @@
 #include "mlir/Dialect/Vector/IR/VectorOps.h"
 #include "mlir/IR/BuiltinTypeInterfaces.h"
 #include "mlir/IR/ImplicitLocOpBuilder.h"
+#include "mlir/IR/Location.h"
 #include "mlir/IR/TypeUtilities.h"
 #include "mlir/Transforms/DialectConversion.h"
+#include "llvm/ADT/SmallVectorExtras.h"
+#include <cstdint>
 
 namespace mlir {
 namespace arith {
@@ -34,6 +37,18 @@ static Value createConst(Location loc, Type type, int value,
   return rewriter.create<arith::ConstantOp>(loc, attr);
 }
 
+/// Create a float constant.
+static Value createFloatConst(Location loc, Type type, APFloat value,
+                              PatternRewriter &rewriter) {
+  auto attr = rewriter.getFloatAttr(getElementTypeOrSelf(type), value);
+  if (auto shapedTy = dyn_cast<ShapedType>(type)) {
+    return rewriter.create<arith::ConstantOp>(
+        loc, DenseElementsAttr::get(shapedTy, attr));
+  }
+
+  return rewriter.create<arith::ConstantOp>(loc, attr);
+}
+
 /// Creates shapedType using shape from cloneFrom and base type from cloneTo
 static Type cloneToShapedType(Type cloneFrom, Type cloneTo) {
   if (auto shapedTy = dyn_cast<ShapedType>(cloneFrom)) {
@@ -322,6 +337,100 @@ struct BFloat16TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
   }
 };
 
+/// In this implementation of extf we take advantage of some key patterns we
+/// notice between the binary representation of an F4E2M1 value and its
+/// corresponding value in F32.
+///
+/// Note: x is sign bit
+/// | Binary | F4E2M1 | f32[23:32]
+/// | x000   | 0.0    | x000 0000 00
+/// | x001   | 0.5    | x011 1111 00
+/// | x010   | 1.0    | x011 1111 10
+/// | x011   | 1.5    | x011 1111 11
+/// | x100   | 2.0    | x010 0000 00
+/// | x101   | 3.0    | x010 0000 01
+/// | x110   | 4.0    | x010 0000 10
+/// | x111   | 6.0    | x010 0000 11
+///
+/// 1) There are only two versions of bits [25:31] in the f32 result
+///     F4E2M1 bits[2:3] decide whether:
+///       - F32 bits[25:31] = 0011 1111
+///       - F32 bits[25:31] = 0010 0000
+///     Exception is zero where
+///       - F32 bits[25:31] = 0000 0000
+///
+/// 2) F4E2M1 bits[1:2] = F32 bits[23:24]
+///     Exception is 0.5 where
+///       - F4E2M1 bits[1:2] = 01, F32 bits[23:24] = 00
+///
+/// 3) F4E2M1 bits[4] = F32 bits[32] (sign bits are equal)
+///
+/// 4) F32 bits[1:22] = 0
+struct F4E2M1ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::ExtFOp op,
+                                PatternRewriter &rewriter) const final {
+    Location loc = op.getLoc();
+    ImplicitLocOpBuilder b(loc, rewriter);
+    Value operand = op.getOperand();
+    Type operandTy = operand.getType();
+    Type resultTy = op.getType();
+    Type operandETy = getElementTypeOrSelf(operandTy);
+    Type resultETy = getElementTypeOrSelf(resultTy);
+
+    if (!isa<Float4E2M1FNType>(operandETy))
+      return rewriter.notifyMatchFailure(op, "not a ext of F4E2M1FN");
+
+    Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
+    Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
+    Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
+    Value i4Bits = b.create<arith::BitcastOp>(i4Ty, operand);
+
+    Value c0x0 = createConst(loc, i4Ty, 0x0, rewriter);
+    Value c0x1 = createConst(loc, i4Ty, 0x1, rewriter);
+    Value c0x2 = createConst(loc, i4Ty, 0x2, rewriter);
+    Value c0x4 = createConst(loc, i4Ty, 0x4, rewriter);
+
+    // Set last Exponent bit and Mantissa.
+    Value c0x00000014 = createConst(loc, i32Ty, 0x14, rewriter);
+    Value bits1To24 = b.create<arith::ShLIOp>(i4Bits, c0x2);
+    Value isHalf =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x1);
+    bits1To24 = b.create<arith::SelectOp>(isHalf, c0x0, bits1To24);
+    bits1To24 = b.create<arith::ExtUIOp>(i32Ty, bits1To24);
+    bits1To24 = b.create<arith::ShLIOp>(bits1To24, c0x00000014);
+
+    // Set first 7 bits of Exponent.
+    Value zeroExpBits = createConst(loc, i32Ty, 0x00000000, rewriter);
+    Value highExpBits = createConst(loc, i32Ty, 0x40000000, rewriter);
+    Value lowExpBits = createConst(loc, i32Ty, 0x3f000000, rewriter);
+    Value useLargerExp =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x4);
+    Value bits25To31 =
+        b.create<arith::SelectOp>(useLargerExp, highExpBits, lowExpBits);
+    Value zeroExp =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, i4Bits, c0x0);
+    bits25To31 = b.create<arith::SelectOp>(zeroExp, zeroExpBits, bits25To31);
+
+    // Set sign.
+    Value c0x80000000 = createConst(loc, i32Ty, 0x80000000, rewriter);
+    Value c0x8 = createConst(loc, i4Ty, 0x8, rewriter);
+    Value negative =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, i4Bits, c0x8);
+    Value bit32 = b.create<arith::SelectOp>(negative, c0x80000000, zeroExpBits);
+
+    // Add segments together.
+    Value bits1To31 = b.create<arith::AddIOp>(bits1To24, bits25To31);
+    Value bits1To32 = b.create<arith::AddIOp>(bits1To31, bit32);
+    Value result = b.create<arith::BitcastOp>(f32Ty, bits1To32);
+    if (!isa<Float32Type>(resultETy))
+      result = b.create<arith::TruncFOp>(resultTy, result);
+
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
   using OpRewritePattern::OpRewritePattern;
   LogicalResult matchAndRewrite(arith::ExtFOp op,
@@ -366,6 +475,130 @@ struct F8E8M0ExtFOpConverter : public OpRewritePattern<arith::ExtFOp> {
   }
 };
 
+/// Conversion from F32 to F4E2M1 according to the OCP Spec:
+/// www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+///
+/// The spec requiers us to perform Round to Nearest, Ties to Even.
+///
+/// This means that after rounding, we should break ties by choosing the option
+/// which results in a mantissa of 0 in the least significant digit.
+///
+/// Table of representable values in F4E2M1:
+///
+/// Note: x is sign bit
+/// | Binary | F4E2M1 | F32[23:32]
+/// | x000   | 0.0    | x000 0000 00
+/// | x001   | 0.5    | x011 1111 00
+/// | x010   | 1.0    | x011 1111 10
+/// | x011   | 1.5    | x011 1111 11
+/// | x100   | 2.0    | x010 0000 00
+/// | x101   | 3.0    | x010 0000 01
+/// | x110   | 4.0    | x010 0000 10
+/// | x111   | 6.0    | x010 0000 11
+///
+/// Conversion procedure:
+///   Step 1: Clamp to representable bounds.
+///   Step 2: Convert exponent by adjusting bias.
+///   Step 3: Set mantissa to first bit.
+///   Step 4: Special consideration for subnormal and zero exponent.
+///   Step 5: Round up if necessary, if mantissa[1:] greater than 1000000 or
+///   subnormal.
+struct F4E2M1TruncFOpConverter : public OpRewritePattern<arith::TruncFOp> {
+  using OpRewritePattern::OpRewritePattern;
+  LogicalResult matchAndRewrite(arith::TruncFOp op,
+                                PatternRewriter &rewriter) const final {
+    Location loc = op.getLoc();
+    ImplicitLocOpBuilder b(loc, rewriter);
+    Value operand = op.getOperand();
+    Type operandTy = operand.getType();
+    Type resultTy = op.getType();
+    Type operandETy = getElementTypeOrSelf(operandTy);
+    Type resultETy = getElementTypeOrSelf(resultTy);
+
+    Type i4Ty = cloneToShapedType(operandTy, b.getI4Type());
+    Type i8Ty = cloneToShapedType(operandTy, b.getI8Type());
+    Type i32Ty = cloneToShapedType(operandTy, b.getI32Type());
+    Type f32Ty = cloneToShapedType(operandTy, b.getF32Type());
+
+    if (!isa<Float32Type>(operandETy))
+      operand = b.create<arith::ExtFOp>(f32Ty, operand);
+    if (!isa<Float4E2M1FNType>(resultETy))
+      return rewriter.notifyMatchFailure(op, "not a trunc of F4E2M1FN");
+
+    Value c0x1 = createConst(loc, i4Ty, 1, rewriter);
+    Value c0x3 = createConst(loc, i4Ty, 3, rewriter);
+    Value c0x00000016 = createConst(loc, i32Ty, 22, rewriter);
+    Value c0x00 = createConst(loc, i8Ty, 0x00, rewriter);
+    Value c0xff = createConst(loc, i8Ty, 0xff, rewriter);
+    Value zeroExpBits = createConst(loc, i32Ty, 0, rewriter);
+
+    // Step 0: Clamp to bounds.
+    Value cHigherBound = createFloatConst(loc, f32Ty, APFloat(6.0f), rewriter);
+    Value cLowerBound = createFloatConst(loc, f32Ty, APFloat(-6.0f), rewriter);
+    Value operandClamped = b.create<arith::MinNumFOp>(cHigherBound, operand);
+    operandClamped = b.create<arith::MaxNumFOp>(cLowerBound, operandClamped);
+    Value f32Bits = b.create<arith::BitcastOp>(i32Ty, operandClamped);
+
+    // Step 1: Set sign bit.
+    Value cF32ExpManWidth = createConst(loc, i32Ty, 31, rewriter); // 23
+    Value f32Sign = b.create<arith::ShRUIOp>(f32Bits, cF32ExpManWidth);
+    Value f4Sign = b.create<arith::TruncIOp>(i4Ty, f32Sign);
+    Value f4Bits = b.create<arith::ShLIOp>(f4Sign, c0x3);
+
+    // Step 2: Convert exponent by adjusting bias.
+    Value biasAdjustment = createConst(loc, i32Ty, 0x7e, rewriter);
+    Value cF4MantissaWidth = c0x1;                                   // 1
+    Value cF32MantissaWidth = createConst(loc, i32Ty, 23, rewriter); // 23
+    Value f32SignExp = b.create<arith::ShRUIOp>(f32Bits, cF32MantissaWidth);
+    Value biasAdjustedSignExp =
+        b.create<arith::SubIOp>(f32SignExp, biasAdjustment);
+    Value f4Exp = b.create<arith::TruncIOp>(i4Ty, biasAdjustedSignExp);
+    f4Exp = b.create<arith::ShLIOp>(f4Exp, cF4MantissaWidth);
+    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Exp);
+
+    // Step 3: Set mantissa to first bit.
+    Value cF32FirstBitMask = createConst(loc, i32Ty, 0x400000, rewriter);
+    Value man1Bit = b.create<arith::AndIOp>(f32Bits, cF32FirstBitMask);
+    man1Bit = b.create<arith::ShRUIOp>(man1Bit, c0x00000016);
+    Value f4Man = b.create<arith::TruncIOp>(i4Ty, man1Bit);
+    f4Bits = b.create<arith::AddIOp>(f4Bits, f4Man);
+
+    // Step 4: Special consideration for conversion to 0.5.
+    Value cF32MantissaMask = createConst(loc, i32Ty, 0x7fffff, rewriter);
+    Value f8Exp = b.create<arith::TruncIOp>(i8Ty, biasAdjustedSignExp);
+    Value isSubnormal =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::sle, f8Exp, c0x00);
+    Value isNegOneExp =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0xff);
+    Value man23Bits = b.create<arith::AndIOp>(f32Bits, cF32MantissaMask);
+    Value isNonZeroMan = b.create<arith::CmpIOp>(arith::CmpIPredicate::ugt,
+                                                 man23Bits, zeroExpBits);
+    Value roundToHalf = b.create<arith::AndIOp>(isNegOneExp, isNonZeroMan);
+    Value isZeroExp =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::eq, f8Exp, c0x00);
+    Value subnormalF4Bits = createConst(loc, i4Ty, 0xf, rewriter);
+    Value halfF4Bits = createConst(loc, i4Ty, 0x0, rewriter);
+    Value subResult =
+        b.create<arith::SelectOp>(isSubnormal, subnormalF4Bits, f4Bits);
+    subResult = b.create<arith::SelectOp>(roundToHalf, halfF4Bits, subResult);
+    f4Bits = b.create<arith::SelectOp>(isZeroExp, f4Bits, subResult);
+
+    // Step 5: Round up if necessary.
+    Value cF32Last22BitMask = createConst(loc, i32Ty, 0x3fffff, rewriter);
+    Value cRound = createConst(loc, i32Ty, 0x200000, rewriter); // 010 0000...
+    Value man22Bits = b.create<arith::AndIOp>(f32Bits, cF32Last22BitMask);
+    Value shouldRound =
+        b.create<arith::CmpIOp>(arith::CmpIPredicate::uge, man22Bits, cRound);
+    shouldRound = b.create<arith::OrIOp>(shouldRound, isSubnormal);
+    Value roundedF4Bits = b.create<arith::AddIOp>(f4Bits, c0x1);
+    f4Bits = b.create<arith::SelectOp>(shouldRound, roundedF4Bits, f4Bits);
+
+    Value result = b.create<arith::BitcastOp>(resultTy, f4Bits);
+    rewriter.replaceOp(op, result);
+    return success();
+  }
+};
+
 /*
 TruncF to F8E8M0 is expected to extract exponent bits out of F32 type
 Since All kinds of Infs and NaNs are mapped to same exponent bits in F32 type,
@@ -498,6 +731,8 @@ struct ArithExpandOpsPass
     arith::populateArithExpandOpsPatterns(patterns);
 
     target.addLegalDialect<arith::ArithDialect>();
+    target.addLegalDialect<vector::VectorDialect>();
+
     // clang-format off
     target.addIllegalOp<
       arith::CeilDivSIOp,
@@ -515,22 +750,24 @@ struct ArithExpandOpsPass
       arith::ScalingTruncFOp
     >();
 
-    if (includeBf16) {
+    if (includeBf16)
       arith::populateExpandBFloat16Patterns(patterns);
-    }
-    if (includeF8E8M0) {
+    if (includeF8E8M0)
       arith::populateExpandF8E8M0Patterns(patterns);
-    }
+    if (includeF4E2M1)
+      arith::populateExpandF4E2M1Patterns(patterns);
 
     target.addDynamicallyLegalOp<arith::ExtFOp>(
       [=](arith::ExtFOp op) {
         Type inETy = getElementTypeOrSelf(op.getOperand().getType());
         Type outETy = getElementTypeOrSelf(op.getType());
         bool legalTypes = true;
-        if (includeBf16) 
+        if (includeBf16)
           legalTypes &= !(inETy.isBF16() && outETy.isF32());
         if (includeF8E8M0)
           legalTypes &= !llvm::isa<Float8E8M0FNUType>(inETy);
+        if (includeF4E2M1)
+          legalTypes &= !llvm::isa<Float4E2M1FNType>(inETy);
         return legalTypes;
       });
 
@@ -539,10 +776,12 @@ struct ArithExpandOpsPass
         Type inETy = getElementTypeOrSelf(op.getOperand().getType());
         Type outETy = getElementTypeOrSelf(op.getType());
         bool legalTypes = true;
-        if (includeBf16) 
+        if (includeBf16)
           legalTypes &= !(inETy.isF32() && outETy.isBF16());
-        if (includeF8E8M0) 
+        if (includeF8E8M0)
           legalTypes &= !(llvm::isa<Float8E8M0FNUType>(outETy)); 
+        if (includeF4E2M1)
+          legalTypes &= !llvm::isa<Float4E2M1FNType>(outETy);
         return legalTypes;
       });
 
@@ -567,6 +806,11 @@ void mlir::arith::populateExpandBFloat16Patterns(RewritePatternSet &patterns) {
       patterns.getContext());
 }
 
+void mlir::arith::populateExpandF4E2M1Patterns(RewritePatternSet &patterns) {
+  patterns.add<F4E2M1ExtFOpConverter, F4E2M1TruncFOpConverter>(
+      patterns.getContext());
+}
+
 void mlir::arith::populateExpandF8E8M0Patterns(RewritePatternSet &patterns) {
   patterns.add<F8E8M0ExtFOpConverter, F8E8M0TruncFOpConverter>(
       patterns.getContext());
diff --git a/mlir/test/Dialect/Arith/expand-ops.mlir b/mlir/test/Dialect/Arith/expand-ops.mlir
index db1349feaff3..8f9b0feba442 100644
--- a/mlir/test/Dialect/Arith/expand-ops.mlir
+++ b/mlir/test/Dialect/Arith/expand-ops.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt %s -arith-expand="include-bf16=true include-f8e8m0=true" -verify-diagnostics -split-input-file | FileCheck %s 
+// RUN: mlir-opt %s -arith-expand="include-bf16=true include-f8e8m0=true include-f4e2m1=true" -verify-diagnostics -split-input-file | FileCheck %s
 // RUN: mlir-opt %s -arith-expand -split-input-file -verify-diagnostics | FileCheck %s --check-prefix=SCHECK
 
 // Test ceil divide with signed integer
@@ -593,3 +593,43 @@ func.func @minui(%a: i32, %b: i32) -> i32 {
 // CHECK-NEXT: %[[CMP:.*]] = arith.cmpi ult, %[[LHS]], %[[RHS]] : i32
 // CHECK-NEXT: %[[RESULT:.*]] = arith.select %[[CMP]], %[[LHS]], %[[RHS]] : i32
 // CHECK-NEXT: return %[[RESULT]] : i32
+
+// -----
+
+func.func @truncf_f32_to_f4E2M1FN(%arg0 : f32) -> f4E2M1FN {
+    %0 = arith.truncf %arg0 : f32 to f4E2M1FN
+    return %0 : f4E2M1FN
+}
+
+// CHECK-LABEL: @truncf_f32_to_f4E2M1FN
+// CHECK-NOT: arith.truncf
+
+// -----
+
+func.func @truncf_vector_f32_to_f4E2M1FN(%arg0 : vector<4xf32>) -> vector<4xf4E2M1FN> {
+    %0 = arith.truncf %arg0 : vector<4xf32> to vector<4xf4E2M1FN>
+    return %0 : vector<4xf4E2M1FN>
+}
+
+// CHECK-LABEL: @truncf_vector_f32_to_f4E2M1FN
+// CHECK-NOT: arith.truncf
+
+// -----
+
+func.func @extf_f4E2M1FN_to_f32(%arg0 : f4E2M1FN) -> f32 {
+    %0 = arith.extf %arg0 : f4E2M1FN to f32
+    return %0 : f32
+}
+
+// CHECK-LABEL: @extf_f4E2M1FN_to_f32
+// CHECK-NOT: arith.extf
+
+// -----
+
+func.func @extf_vector_f4E2M1FN_to_f32(%arg0 : vector<4xf4E2M1FN>) -> vector<4xf32> {
+    %0 = arith.extf %arg0 : vector<4xf4E2M1FN> to vector<4xf32>
+    return %0 : vector<4xf32>
+}
+
+// CHECK-LABEL: @extf_vector_f4E2M1FN_to_f32
+// CHECK-NOT: arith.extf
diff --git a/mlir/test/Integration/Dialect/Arith/CPU/test-arith-expand-truncf-extf.mlir b/mlir/test/Integration/Dialect/Arith/CPU/test-arith-expand-truncf-extf.mlir
new file mode 100644
index 000000000000..9c310d80d4c2
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Arith/CPU/test-arith-expand-truncf-extf.mlir
@@ -0,0 +1,73 @@
+// Check various edge cases for truncf/extf ops involving f32 and f4e2m1 types.
+
+// RUN: mlir-opt %s --convert-func-to-llvm \
+// RUN:             --arith-expand="include-f4e2m1=true" \
+// RUN:             --convert-arith-to-llvm --convert-vector-to-llvm \
+// RUN:             --reconcile-unrealized-casts | \
+// RUN:   mlir-runner -e entry --entry-point-result=void \
+// RUN:               --shared-libs=%mlir_c_runner_utils | \
+// RUN:   FileCheck %s --match-full-lines
+
+func.func @check_extf(%in : f4E2M1FN) -> () {
+  %res = arith.extf %in : f4E2M1FN to f32
+  vector.print %res : f32
+  return
+}
+
+// See https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf
+// for details on F4E2M1 representation 
+func.func @check_truncf(%in : f32) -> () {
+  %trunc = arith.truncf %in : f32 to f4E2M1FN
+  %bitcast = arith.bitcast %trunc : f4E2M1FN to i4
+  %res = arith.extui %bitcast : i4 to i64
+  vector.print %res : i64
+  return
+}
+
+func.func @entry() {
+  %zero = arith.constant 0.0 : f32
+  %half = arith.constant 0.5 : f32
+  %one = arith.constant 1.0 : f32
+  %max = arith.constant 6.0 : f32
+  %min = arith.constant -6.0 : f32
+  %lowerThanMin = arith.constant -1000000.0 : f32
+  %higherThanMax = arith.constant 1000000.0 : f32
+  %mustRound = arith.constant -3.14 : f32
+  %nan = arith.constant 0x7f80000 : f32
+
+  // CHECK: 0
+  func.call @check_truncf(%zero) : (f32) -> ()
+  // CHECK: 1
+  func.call @check_truncf(%half) : (f32) -> ()
+  // CHECK: 2
+  func.call @check_truncf(%one) : (f32) -> ()
+  // CHECK: 7
+  func.call @check_truncf(%max) : (f32) -> ()
+  // CHECK: 15
+  func.call @check_truncf(%min) : (f32) -> ()
+  // CHECK: 7
+  func.call @check_truncf(%higherThanMax) : (f32) -> ()
+  // CHECK: 15
+  func.call @check_truncf(%lowerThanMin) : (f32) -> ()
+  // CHECK: 13
+  func.call @check_truncf(%mustRound) : (f32) -> ()
+  // CHECK: 0
+  func.call @check_truncf(%nan) : (f32) -> ()
+
+  // CHECK: 0
+  %zeroF4 = arith.truncf %zero : f32 to f4E2M1FN
+  func.call @check_extf(%zeroF4) : (f4E2M1FN) -> ()
+  // CHECK: 0.5
+  %halfF4 = arith.truncf %half : f32 to f4E2M1FN
+  func.call @check_extf(%halfF4) : (f4E2M1FN) -> ()
+  // CHECK: 6
+  %higherThanMaxF4 = arith.truncf %higherThanMax : f32 to f4E2M1FN
+  func.call @check_extf(%higherThanMaxF4) : (f4E2M1FN) -> ()
+  // CHECK: -6
+  %lowerThanMinF4 = arith.truncf %lowerThanMin : f32 to f4E2M1FN
+  func.call @check_extf(%lowerThanMinF4) : (f4E2M1FN) -> ()
+  // CHECK: -3
+  %mustRoundF4 = arith.truncf %mustRound : f32 to f4E2M1FN
+  func.call @check_extf(%mustRoundF4) : (f4E2M1FN) -> ()
+  return
+}
-- 
cgit v1.2.3


From edbaf19c46c678e080d5a43e719bc19a02b44ca9 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Fri, 20 Jun 2025 12:29:32 -0400
Subject: [AMDGPU] Fix a potential integer overflow in GCNRegPressure when
 true16 is enabled (#144968)

Fixes SWDEV-537014.
---
 llvm/lib/Target/AMDGPU/GCNRegPressure.cpp          | 28 +++++++++++--
 .../gcn-reg-pressure-true16-integer-overflow.mir   | 48 ++++++++++++++++++++++
 2 files changed, 72 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/gcn-reg-pressure-true16-integer-overflow.mir

diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
index ce213b91b1f7..5724ce9cc5d1 100644
--- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp
@@ -44,16 +44,19 @@ void GCNRegPressure::inc(unsigned Reg,
                          LaneBitmask PrevMask,
                          LaneBitmask NewMask,
                          const MachineRegisterInfo &MRI) {
-  if (SIRegisterInfo::getNumCoveredRegs(NewMask) ==
-      SIRegisterInfo::getNumCoveredRegs(PrevMask))
+  unsigned NewNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(NewMask);
+  unsigned PrevNumCoveredRegs = SIRegisterInfo::getNumCoveredRegs(PrevMask);
+  if (NewNumCoveredRegs == PrevNumCoveredRegs)
     return;
 
   int Sign = 1;
   if (NewMask < PrevMask) {
     std::swap(NewMask, PrevMask);
+    std::swap(NewNumCoveredRegs, PrevNumCoveredRegs);
     Sign = -1;
   }
-  assert(PrevMask < NewMask && "prev mask should always be lesser than new");
+  assert(PrevMask < NewMask && PrevNumCoveredRegs < NewNumCoveredRegs &&
+         "prev mask should always be lesser than new");
 
   const TargetRegisterClass *RC = MRI.getRegClass(Reg);
   const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
@@ -66,7 +69,24 @@ void GCNRegPressure::inc(unsigned Reg,
       Value[TupleIdx] += Sign * TRI->getRegClassWeight(RC).RegWeight;
     }
     // Pressure scales with number of new registers covered by the new mask.
-    Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
+    // Note when true16 is enabled, we can no longer safely use the following
+    // approach to calculate the difference in the number of 32-bit registers
+    // between two masks:
+    //
+    // Sign *= SIRegisterInfo::getNumCoveredRegs(~PrevMask & NewMask);
+    //
+    // The issue is that the mask calculation `~PrevMask & NewMask` doesn't
+    // properly account for partial usage of a 32-bit register when dealing with
+    // 16-bit registers.
+    //
+    // Consider this example:
+    // Assume PrevMask = 0b0010 and NewMask = 0b1111. Here, the correct register
+    // usage difference should be 1, because even though PrevMask uses only half
+    // of a 32-bit register, it should still be counted as a full register use.
+    // However, the mask calculation yields `~PrevMask & NewMask = 0b1101`, and
+    // calling `getNumCoveredRegs` returns 2 instead of 1. This incorrect
+    // calculation can lead to integer overflow when Sign = -1.
+    Sign *= NewNumCoveredRegs - PrevNumCoveredRegs;
   }
   Value[RegKind] += Sign;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/gcn-reg-pressure-true16-integer-overflow.mir b/llvm/test/CodeGen/AMDGPU/gcn-reg-pressure-true16-integer-overflow.mir
new file mode 100644
index 000000000000..7f0654746e13
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/gcn-reg-pressure-true16-integer-overflow.mir
@@ -0,0 +1,48 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -x mir -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1102 -run-pass=machine-scheduler %s -o - | FileCheck %s
+
+---
+name:            foo
+tracksRegLiveness: true
+liveins:
+  - { reg: '$sgpr4_sgpr5', virtual-reg: '%0' }
+body:             |
+  bb.0.entry:
+    liveins: $sgpr4_sgpr5
+
+    ; CHECK-LABEL: name: foo
+    ; CHECK: liveins: $sgpr4_sgpr5
+    ; CHECK-NEXT: {{  $}}
+    ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+    ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+    ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub0:sgpr_128 = COPY [[S_MOV_B32_]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]]
+    ; CHECK-NEXT: [[COPY1:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]]
+    ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX2_OFFSET:%[0-9]+]]:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET [[COPY1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+    ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+    ; CHECK-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64 = V_LSHRREV_B64_e64 24, [[BUFFER_LOAD_DWORDX2_OFFSET]], implicit $exec
+    ; CHECK-NEXT: undef [[COPY2:%[0-9]+]].lo16:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].lo16
+    ; CHECK-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 16, [[COPY2]], implicit $exec
+    ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[S_LOAD_DWORDX2_IMM]]
+    ; CHECK-NEXT: [[V_PK_LSHLREV_B16_:%[0-9]+]]:vgpr_32 = V_PK_LSHLREV_B16 0, 8, 8, [[V_LSHLREV_B32_e64_]], 0, 0, 0, 0, 0, implicit $exec
+    ; CHECK-NEXT: FLAT_STORE_DWORD [[COPY3]], [[V_PK_LSHLREV_B16_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (s32))
+    ; CHECK-NEXT: S_WAITCNT 0
+    ; CHECK-NEXT: S_ENDPGM 0
+    %0:sgpr_64(p4) = COPY killed $sgpr4_sgpr5
+    %1:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed %0(p4), 0, 0 :: (dereferenceable invariant load (s64), align 16, addrspace 4)
+    %2:sreg_32 = S_MOV_B32 0
+    undef %3.sub0:sgpr_128 = COPY %2
+    %3.sub1:sgpr_128 = COPY %2
+    %3.sub2:sgpr_128 = COPY %2
+    %3.sub3:sgpr_128 = COPY killed %2
+    %4:vreg_64 = BUFFER_LOAD_DWORDX2_OFFSET killed %3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s64), align 1, addrspace 8)
+    %5:vreg_64 = V_LSHRREV_B64_e64 24, killed %4, implicit $exec
+    undef %6.lo16:vgpr_32 = COPY killed %5.lo16
+    %7:vgpr_32 = V_LSHLREV_B32_e64 16, killed %6, implicit $exec
+    %8:vgpr_32 = V_PK_LSHLREV_B16 0, 8, 8, killed %7, 0, 0, 0, 0, 0, implicit $exec
+    %9:vreg_64 = COPY killed %1
+    FLAT_STORE_DWORD killed %9, killed %8, 0, 0, implicit $exec, implicit $flat_scr :: (store (s32))
+    S_WAITCNT 0
+    S_ENDPGM 0
+...
-- 
cgit v1.2.3


From 945ce1aa3d29e24c49720ae9e0bcfbac88f2defd Mon Sep 17 00:00:00 2001
From: Hristo Hristov <hghristov.rmm@gmail.com>
Date: Fri, 20 Jun 2025 19:36:46 +0300
Subject: [libc++] Update the value of __cpp_lib_constrained_equality after
 P3379R0 (#144553)

https://wg21.link/P3379R0 updated the value of __cpp_lib_constrained_equality,
but we forgot to update it when we implemented the paper.
---
 libcxx/include/version                             |  6 ++--
 .../expected.version.compile.pass.cpp              | 33 ++++++++++++++++++++++
 .../optional.version.compile.pass.cpp              |  4 +--
 .../tuple.version.compile.pass.cpp                 |  4 +--
 .../utility.version.compile.pass.cpp               |  4 +--
 .../variant.version.compile.pass.cpp               |  4 +--
 .../version.version.compile.pass.cpp               |  4 +--
 .../generate_feature_test_macro_components.py      |  7 +++--
 8 files changed, 51 insertions(+), 15 deletions(-)

diff --git a/libcxx/include/version b/libcxx/include/version
index 7154cab92335..f430c7edff2b 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -83,8 +83,8 @@ __cpp_lib_constexpr_tuple                               201811L <tuple>
 __cpp_lib_constexpr_typeinfo                            202106L <typeinfo>
 __cpp_lib_constexpr_utility                             201811L <utility>
 __cpp_lib_constexpr_vector                              201907L <vector>
-__cpp_lib_constrained_equality                          202403L <optional> <tuple> <utility>
-                                                                <variant>
+__cpp_lib_constrained_equality                          202411L <expected> <optional> <tuple>
+                                                                <utility> <variant>
 __cpp_lib_containers_ranges                             202202L <deque> <forward_list> <list>
                                                                 <map> <queue> <set>
                                                                 <stack> <string> <unordered_map>
@@ -551,7 +551,7 @@ __cpp_lib_void_t                                        201411L <type_traits>
 #   define __cpp_lib_constexpr_new                      202406L
 # endif
 # define __cpp_lib_constexpr_queue                      202502L
-// # define __cpp_lib_constrained_equality                 202403L
+// # define __cpp_lib_constrained_equality                 202411L
 // # define __cpp_lib_copyable_function                    202306L
 // # define __cpp_lib_debugging                            202311L
 // # define __cpp_lib_default_template_type_for_algorithm_values 202403L
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/expected.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/expected.version.compile.pass.cpp
index d58f726f66e2..9c7a84f145dd 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/expected.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/expected.version.compile.pass.cpp
@@ -20,6 +20,10 @@
 
 #if TEST_STD_VER < 14
 
+#  ifdef __cpp_lib_constrained_equality
+#    error "__cpp_lib_constrained_equality should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_expected
 #    error "__cpp_lib_expected should not be defined before c++23"
 #  endif
@@ -30,6 +34,10 @@
 
 #elif TEST_STD_VER == 14
 
+#  ifdef __cpp_lib_constrained_equality
+#    error "__cpp_lib_constrained_equality should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_expected
 #    error "__cpp_lib_expected should not be defined before c++23"
 #  endif
@@ -40,6 +48,10 @@
 
 #elif TEST_STD_VER == 17
 
+#  ifdef __cpp_lib_constrained_equality
+#    error "__cpp_lib_constrained_equality should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_expected
 #    error "__cpp_lib_expected should not be defined before c++23"
 #  endif
@@ -50,6 +62,10 @@
 
 #elif TEST_STD_VER == 20
 
+#  ifdef __cpp_lib_constrained_equality
+#    error "__cpp_lib_constrained_equality should not be defined before c++26"
+#  endif
+
 #  ifdef __cpp_lib_expected
 #    error "__cpp_lib_expected should not be defined before c++23"
 #  endif
@@ -60,6 +76,10 @@
 
 #elif TEST_STD_VER == 23
 
+#  ifdef __cpp_lib_constrained_equality
+#    error "__cpp_lib_constrained_equality should not be defined before c++26"
+#  endif
+
 #  ifndef __cpp_lib_expected
 #    error "__cpp_lib_expected should be defined in c++23"
 #  endif
@@ -73,6 +93,19 @@
 
 #elif TEST_STD_VER > 23
 
+#  if !defined(_LIBCPP_VERSION)
+#    ifndef __cpp_lib_constrained_equality
+#      error "__cpp_lib_constrained_equality should be defined in c++26"
+#    endif
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
+#    endif
+#  else
+#    ifdef __cpp_lib_constrained_equality
+#      error "__cpp_lib_constrained_equality should not be defined because it is unimplemented in libc++!"
+#    endif
+#  endif
+
 #  ifndef __cpp_lib_expected
 #    error "__cpp_lib_expected should be defined in c++26"
 #  endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
index f8ff69f61877..32685972d601 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp
@@ -123,8 +123,8 @@
 #    ifndef __cpp_lib_constrained_equality
 #      error "__cpp_lib_constrained_equality should be defined in c++26"
 #    endif
-#    if __cpp_lib_constrained_equality != 202403L
-#      error "__cpp_lib_constrained_equality should have the value 202403L in c++26"
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
 #    endif
 #  else
 #    ifdef __cpp_lib_constrained_equality
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.compile.pass.cpp
index f4a2fd8b29f4..b583edfc43ad 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/tuple.version.compile.pass.cpp
@@ -274,8 +274,8 @@
 #    ifndef __cpp_lib_constrained_equality
 #      error "__cpp_lib_constrained_equality should be defined in c++26"
 #    endif
-#    if __cpp_lib_constrained_equality != 202403L
-#      error "__cpp_lib_constrained_equality should have the value 202403L in c++26"
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
 #    endif
 #  else
 #    ifdef __cpp_lib_constrained_equality
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/utility.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/utility.version.compile.pass.cpp
index e0a83c7813b2..7dd347857633 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/utility.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/utility.version.compile.pass.cpp
@@ -405,8 +405,8 @@
 #    ifndef __cpp_lib_constrained_equality
 #      error "__cpp_lib_constrained_equality should be defined in c++26"
 #    endif
-#    if __cpp_lib_constrained_equality != 202403L
-#      error "__cpp_lib_constrained_equality should have the value 202403L in c++26"
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
 #    endif
 #  else
 #    ifdef __cpp_lib_constrained_equality
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
index dc4af4d09f9e..4a7b9f7431a8 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/variant.version.compile.pass.cpp
@@ -103,8 +103,8 @@
 #    ifndef __cpp_lib_constrained_equality
 #      error "__cpp_lib_constrained_equality should be defined in c++26"
 #    endif
-#    if __cpp_lib_constrained_equality != 202403L
-#      error "__cpp_lib_constrained_equality should have the value 202403L in c++26"
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
 #    endif
 #  else
 #    ifdef __cpp_lib_constrained_equality
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index e4fe9f994e2e..cef334f70c07 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -6575,8 +6575,8 @@
 #    ifndef __cpp_lib_constrained_equality
 #      error "__cpp_lib_constrained_equality should be defined in c++26"
 #    endif
-#    if __cpp_lib_constrained_equality != 202403L
-#      error "__cpp_lib_constrained_equality should have the value 202403L in c++26"
+#    if __cpp_lib_constrained_equality != 202411L
+#      error "__cpp_lib_constrained_equality should have the value 202411L in c++26"
 #    endif
 #  else
 #    ifdef __cpp_lib_constrained_equality
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index de06b9dd1bee..849a96b9585b 100644
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -431,8 +431,11 @@ feature_test_macros = [
         },
         {
             "name": "__cpp_lib_constrained_equality",
-            "values": {"c++26": 202403}, # P2944R3: Comparisons for reference_wrapper
-            "headers": ["optional", "tuple", "utility", "variant"],
+            "values": {
+                # "c++26": 202403,  # P2944R3: Comparisons for reference_wrapper
+                "c++26": 202411,  # P3379R0: Constrain std::expected equality operators
+            },
+            "headers": ["expected", "optional", "tuple", "utility", "variant"],
             "unimplemented": True,
         },
         {
-- 
cgit v1.2.3


From 6ddb3a69c15963b7cf1cbb0323dd3e467b71cefc Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Fri, 20 Jun 2025 17:54:44 +0100
Subject: [AMDGPU] Add another test showing unwanted VALU codegen (#145062)

---
 .../CodeGen/AMDGPU/pseudo-scalar-transcendental.ll | 53 +++++++++++++++-------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index a2e7f2e62f5d..d957aaa8a48a 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -410,19 +410,40 @@ define amdgpu_cs half @srcmods_neg_f16(half inreg %src) {
   ret half %result
 }
 
-declare half @llvm.exp2.f16(half)
-declare float @llvm.exp2.f32(float)
-declare half @llvm.amdgcn.exp2.f16(half)
-declare float @llvm.amdgcn.exp2.f32(float)
-declare half @llvm.log2.f16(half)
-declare float @llvm.log2.f32(float)
-declare half @llvm.amdgcn.log.f16(half)
-declare float @llvm.amdgcn.log.f32(float)
-declare half @llvm.amdgcn.rcp.f16(half)
-declare float @llvm.amdgcn.rcp.f32(float)
-declare half @llvm.sqrt.f16(half)
-declare float @llvm.sqrt.f32(float)
-declare half @llvm.amdgcn.sqrt.f16(half)
-declare float @llvm.amdgcn.sqrt.f32(float)
-declare half @llvm.fabs.f16(half)
-declare float @llvm.fabs.f32(float)
+; TODO: SelectionDAG should avoid generating v_rcp_iflag_f32.
+define amdgpu_cs float @fdiv_f32_i32(float inreg %a, i32 inreg %b) {
+; GFX12-SDAG-LABEL: fdiv_f32_i32:
+; GFX12-SDAG:       ; %bb.0:
+; GFX12-SDAG-NEXT:    s_cvt_f32_u32 s1, s1
+; GFX12-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT:    v_rcp_iflag_f32_e32 v0, s1
+; GFX12-SDAG-NEXT:    v_mul_f32_e32 v0, s0, v0
+; GFX12-SDAG-NEXT:    ; return to shader part epilog
+;
+; GFX12-GISEL-LABEL: fdiv_f32_i32:
+; GFX12-GISEL:       ; %bb.0:
+; GFX12-GISEL-NEXT:    s_cvt_f32_u32 s1, s1
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-GISEL-NEXT:    v_s_rcp_f32 s1, s1
+; GFX12-GISEL-NEXT:    s_mul_f32 s0, s0, s1
+; GFX12-GISEL-NEXT:    s_wait_alu 0xfffe
+; GFX12-GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT:    ; return to shader part epilog
+  %uint = uitofp i32 %b to float
+  %result = fdiv afn float %a, %uint
+  ret float %result
+}
+
+define amdgpu_cs half @fdiv_f16_i16(half inreg %a, i16 inreg %b) {
+; GFX12-LABEL: fdiv_f16_i16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_cvt_f16_u16_e32 v0, s1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX12-NEXT:    v_rcp_f16_e32 v0, v0
+; GFX12-NEXT:    v_mul_f16_e32 v0, s0, v0
+; GFX12-NEXT:    ; return to shader part epilog
+  %uint = uitofp i16 %b to half
+  %result = fdiv afn half %a, %uint
+  ret half %result
+}
-- 
cgit v1.2.3


From 7157f33c6c88143acb8bc02bc26a0d6786136419 Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Fri, 20 Jun 2025 09:58:59 -0700
Subject: [libc++] Disable a std::unexpected test in modules build (#144466)

This patch disables unexpected_disabled_cpp17.verify.cpp under clang
modules builds because it changes diagnostics criteria post #143423,
causing the test to fail.

This patch follows a similar style to 853059a15011fd8b57dd0.
This was found when working on trying to land #144033.
---
 .../depr/exception.unexpected/unexpected_disabled_cpp17.verify.cpp   | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libcxx/test/libcxx/depr/exception.unexpected/unexpected_disabled_cpp17.verify.cpp b/libcxx/test/libcxx/depr/exception.unexpected/unexpected_disabled_cpp17.verify.cpp
index 299cb123fbda..b92f2baa02e4 100644
--- a/libcxx/test/libcxx/depr/exception.unexpected/unexpected_disabled_cpp17.verify.cpp
+++ b/libcxx/test/libcxx/depr/exception.unexpected/unexpected_disabled_cpp17.verify.cpp
@@ -8,6 +8,11 @@
 
 // UNSUPPORTED: c++03, c++11, c++14
 
+// When built with modules, this test gives diagnostics like declaration of
+// 'unexpected' must be imported from module 'std.expected.unexpected' before
+// it is required. Therefore disable it in this configuration.
+// UNSUPPORTED: clang-modules-build
+
 // test unexpected
 
 #include <exception>
-- 
cgit v1.2.3


From 8d6e29d0d3e3acb67adaa61648f35a38a9d196b3 Mon Sep 17 00:00:00 2001
From: Uzair Nawaz <uzairnawaz@google.com>
Date: Fri, 20 Jun 2025 16:59:30 +0000
Subject: [libc] Reworked CharacterConverter isComplete into isFull and isEmpty
 (#144799)

isComplete previously meant different things for different conversion
directions.
Refactored bytes_processed to bytes_stored which now consistently
increments on every push and decrements on pop making both directions
more consistent with each other
---
 libc/src/__support/wchar/character_converter.cpp  | 33 ++++++++--------
 libc/src/__support/wchar/character_converter.h    |  3 +-
 libc/src/__support/wchar/mbstate.h                |  6 +--
 libc/test/src/__support/wchar/utf32_to_8_test.cpp | 48 +++++++++++++----------
 libc/test/src/__support/wchar/utf8_to_32_test.cpp | 20 +++++-----
 5 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/libc/src/__support/wchar/character_converter.cpp b/libc/src/__support/wchar/character_converter.cpp
index 5ab0447bb08b..1f81de4248ff 100644
--- a/libc/src/__support/wchar/character_converter.cpp
+++ b/libc/src/__support/wchar/character_converter.cpp
@@ -30,18 +30,20 @@ CharacterConverter::CharacterConverter(mbstate *mbstate) { state = mbstate; }
 
 void CharacterConverter::clear() {
   state->partial = 0;
-  state->bytes_processed = 0;
+  state->bytes_stored = 0;
   state->total_bytes = 0;
 }
 
-bool CharacterConverter::isComplete() {
-  return state->bytes_processed == state->total_bytes;
+bool CharacterConverter::isFull() {
+  return state->bytes_stored == state->total_bytes && state->total_bytes != 0;
 }
 
+bool CharacterConverter::isEmpty() { return state->bytes_stored == 0; }
+
 int CharacterConverter::push(char8_t utf8_byte) {
   uint8_t num_ones = static_cast<uint8_t>(cpp::countl_one(utf8_byte));
   // Checking the first byte if first push
-  if (state->bytes_processed == 0) {
+  if (isEmpty()) {
     // UTF-8 char has 1 byte total
     if (num_ones == 0) {
       state->total_bytes = 1;
@@ -58,21 +60,21 @@ int CharacterConverter::push(char8_t utf8_byte) {
     }
     // Invalid first byte
     else {
-      // bytes_processed and total_bytes will always be 0 here
+      // bytes_stored and total_bytes will always be 0 here
       state->partial = static_cast<char32_t>(0);
       return -1;
     }
     state->partial = static_cast<char32_t>(utf8_byte);
-    state->bytes_processed++;
+    state->bytes_stored++;
     return 0;
   }
   // Any subsequent push
   // Adding 6 more bits so need to left shift
-  if (num_ones == 1 && !isComplete()) {
+  if (num_ones == 1 && !isFull()) {
     char32_t byte = utf8_byte & MASK_ENCODED_BITS;
     state->partial = state->partial << ENCODED_BITS_PER_UTF8;
     state->partial |= byte;
-    state->bytes_processed++;
+    state->bytes_stored++;
     return 0;
   }
   // Invalid byte -> reset the state
@@ -82,11 +84,10 @@ int CharacterConverter::push(char8_t utf8_byte) {
 
 int CharacterConverter::push(char32_t utf32) {
   // we can't be partially through a conversion when pushing a utf32 value
-  if (!isComplete())
+  if (!isEmpty())
     return -1;
 
   state->partial = utf32;
-  state->bytes_processed = 0;
 
   // determine number of utf-8 bytes needed to represent this utf32 value
   constexpr char32_t MAX_VALUE_PER_UTF8_LEN[] = {0x7f, 0x7ff, 0xffff, 0x10ffff};
@@ -94,6 +95,7 @@ int CharacterConverter::push(char32_t utf32) {
   for (uint8_t i = 0; i < NUM_RANGES; i++) {
     if (state->partial <= MAX_VALUE_PER_UTF8_LEN[i]) {
       state->total_bytes = i + 1;
+      state->bytes_stored = i + 1;
       return 0;
     }
   }
@@ -107,7 +109,7 @@ int CharacterConverter::push(char32_t utf32) {
 ErrorOr<char32_t> CharacterConverter::pop_utf32() {
   // If pop is called too early, do not reset the state, use error to determine
   // whether enough bytes have been pushed
-  if (!isComplete() || state->bytes_processed == 0)
+  if (!isFull())
     return Error(-1);
   char32_t utf32 = state->partial;
   // reset if successful pop
@@ -116,7 +118,7 @@ ErrorOr<char32_t> CharacterConverter::pop_utf32() {
 }
 
 ErrorOr<char8_t> CharacterConverter::pop_utf8() {
-  if (isComplete())
+  if (isEmpty())
     return Error(-1);
 
   constexpr char8_t FIRST_BYTE_HEADERS[] = {0, 0xC0, 0xE0, 0xF0};
@@ -125,9 +127,8 @@ ErrorOr<char8_t> CharacterConverter::pop_utf8() {
   char32_t output;
 
   // Shift to get the next 6 bits from the utf32 encoding
-  const size_t shift_amount =
-      (state->total_bytes - state->bytes_processed - 1) * ENCODED_BITS_PER_UTF8;
-  if (state->bytes_processed == 0) {
+  const size_t shift_amount = (state->bytes_stored - 1) * ENCODED_BITS_PER_UTF8;
+  if (isFull()) {
     /*
       Choose the correct set of most significant bits to encode the length
       of the utf8 sequence. The remaining bits contain the most significant
@@ -141,7 +142,7 @@ ErrorOr<char8_t> CharacterConverter::pop_utf8() {
              ((state->partial >> shift_amount) & MASK_ENCODED_BITS);
   }
 
-  state->bytes_processed++;
+  state->bytes_stored--;
   return static_cast<char8_t>(output);
 }
 
diff --git a/libc/src/__support/wchar/character_converter.h b/libc/src/__support/wchar/character_converter.h
index c4ba7cf6b689..be0e6129df23 100644
--- a/libc/src/__support/wchar/character_converter.h
+++ b/libc/src/__support/wchar/character_converter.h
@@ -26,7 +26,8 @@ public:
   CharacterConverter(mbstate *mbstate);
 
   void clear();
-  bool isComplete();
+  bool isFull();
+  bool isEmpty();
 
   int push(char8_t utf8_byte);
   int push(char32_t utf32);
diff --git a/libc/src/__support/wchar/mbstate.h b/libc/src/__support/wchar/mbstate.h
index fb08fb4eaa18..fea693f73c3b 100644
--- a/libc/src/__support/wchar/mbstate.h
+++ b/libc/src/__support/wchar/mbstate.h
@@ -22,10 +22,10 @@ struct mbstate {
 
   /*
   Progress towards a conversion
-    For utf8  -> utf32, increases with each CharacterConverter::push(utf8_byte)
-    For utf32 ->  utf8, increases with each CharacterConverter::pop_utf8()
+    Increases with each push(...) until it reaches total_bytes
+    Decreases with each pop(...) until it reaches 0
   */
-  uint8_t bytes_processed;
+  uint8_t bytes_stored;
 
   // Total number of bytes that will be needed to represent this character
   uint8_t total_bytes;
diff --git a/libc/test/src/__support/wchar/utf32_to_8_test.cpp b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
index f4c5cb863ff3..a6a7bc4aa6f4 100644
--- a/libc/test/src/__support/wchar/utf32_to_8_test.cpp
+++ b/libc/test/src/__support/wchar/utf32_to_8_test.cpp
@@ -20,17 +20,19 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, OneByte) {
   // utf8 1-byte encodings are identical to their utf32 representations
   char32_t utf32_A = 0x41; // 'A'
   cr.push(utf32_A);
+  ASSERT_TRUE(cr.isFull());
   auto popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<char>(popped.value()), 'A');
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   char32_t utf32_B = 0x42; // 'B'
   cr.push(utf32_B);
+  ASSERT_TRUE(cr.isFull());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<char>(popped.value()), 'B');
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // should error if we try to pop another utf8 byte out
   popped = cr.pop_utf8();
@@ -45,26 +47,28 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, TwoByte) {
   // testing utf32: 0xff -> utf8: 0xc3 0xbf
   char32_t utf32 = 0xff;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   auto popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xc3);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xbf);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // testing utf32: 0x58e -> utf8: 0xd6 0x8e
   utf32 = 0x58e;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xd6);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x8e);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // should error if we try to pop another utf8 byte out
   popped = cr.pop_utf8();
@@ -79,34 +83,36 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, ThreeByte) {
   // testing utf32: 0xac15 -> utf8: 0xea 0xb0 0x95
   char32_t utf32 = 0xac15;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   auto popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xea);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xb0);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x95);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // testing utf32: 0x267b -> utf8: 0xe2 0x99 0xbb
   utf32 = 0x267b;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xe2);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x99);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xbb);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // should error if we try to pop another utf8 byte out
   popped = cr.pop_utf8();
@@ -121,42 +127,44 @@ TEST(LlvmLibcCharacterConverterUTF32To8Test, FourByte) {
   // testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
   char32_t utf32 = 0x1f921;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   auto popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x9f);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xa4);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // testing utf32: 0x12121 -> utf8: 0xf0 0x92 0x84 0xa1
   utf32 = 0x12121;
   cr.push(utf32);
+  ASSERT_TRUE(cr.isFull());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xf0);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x92);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0x84);
-  ASSERT_TRUE(!cr.isComplete());
+  ASSERT_TRUE(!cr.isEmpty());
   popped = cr.pop_utf8();
   ASSERT_TRUE(popped.has_value());
   ASSERT_EQ(static_cast<int>(popped.value()), 0xa1);
-  ASSERT_TRUE(cr.isComplete());
+  ASSERT_TRUE(cr.isEmpty());
 
   // should error if we try to pop another utf8 byte out
   popped = cr.pop_utf8();
diff --git a/libc/test/src/__support/wchar/utf8_to_32_test.cpp b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
index 9cb059faa937..36ae7d689cc0 100644
--- a/libc/test/src/__support/wchar/utf8_to_32_test.cpp
+++ b/libc/test/src/__support/wchar/utf8_to_32_test.cpp
@@ -13,7 +13,7 @@
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, OneByte) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   char ch = 'A';
 
@@ -28,7 +28,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, OneByte) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[2] = {static_cast<char>(0xC2),
                       static_cast<char>(0x8E)}; // Ž car symbol
@@ -44,7 +44,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoBytes) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
                       static_cast<char>(0x91)}; // ∑ sigma symbol
@@ -61,7 +61,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ThreeBytes) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
                       static_cast<char>(0xA4),
@@ -80,7 +80,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, FourBytes) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidByte) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch = static_cast<char>(0x80); // invalid starting bit sequence
 
@@ -92,7 +92,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidByte) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[4] = {
       static_cast<char>(0x80), static_cast<char>(0x00), static_cast<char>(0x80),
@@ -112,7 +112,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidMultiByte) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidLastByte) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   // Last byte is invalid since it does not have correct starting sequence.
   // 0xC0 --> 11000000 starting sequence should be 10xxxxxx
@@ -132,7 +132,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidLastByte) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0x80)};
@@ -153,7 +153,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, ValidTwoByteWithExtraRead) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
                       static_cast<char>(0xC7), static_cast<char>(0x8C)};
@@ -179,7 +179,7 @@ TEST(LlvmLibcCharacterConverterUTF8To32Test, TwoValidTwoBytes) {
 
 TEST(LlvmLibcCharacterConverterUTF8To32Test, InvalidPop) {
   LIBC_NAMESPACE::internal::mbstate state;
-  state.bytes_processed = 0;
+  state.bytes_stored = 0;
   state.total_bytes = 0;
   LIBC_NAMESPACE::internal::CharacterConverter char_conv(&state);
   const char ch[2] = {static_cast<char>(0xC2), static_cast<char>(0x8E)};
-- 
cgit v1.2.3


From 65cb3bcf327da8f9740e56897bc9954364e59eb6 Mon Sep 17 00:00:00 2001
From: Maryam Moghadas <maryammo@ca.ibm.com>
Date: Fri, 20 Jun 2025 13:03:14 -0400
Subject: [Clang][PowerPC] Add __dmr1024 type and DMF integer calculation
 builtins (#142480)

Define the __dmr1024 type used to manipulate the new DMR registers
introduced by the Dense Math Facility (DMF) on PowerPC, and add six
Clang builtins that correspond to the integer outer-product accumulate
to ACC PowerPC instructions:
*  __builtin_mma_dmxvi8gerx4
* __builtin_mma_pmdmxvi8gerx4
*  __builtin_mma_dmxvi8gerx4pp
*  __builtin_mma_pmdmxvi8gerx4pp
*  __builtin_mma_dmxvi8gerx4spp
* __builtin_mma_pmdmxvi8gerx4spp.
---
 clang/include/clang/Basic/BuiltinsPPC.def          |  12 ++
 clang/include/clang/Basic/PPCTypes.def             |   1 +
 clang/lib/AST/ASTContext.cpp                       |   1 +
 clang/test/AST/ast-dump-ppc-types.c                |  13 +-
 clang/test/CodeGen/PowerPC/builtins-ppc-dmf.c      |  94 +++++++++++
 .../ppc-dmf-paired-vec-memops-builtin-err.c        |  20 +++
 clang/test/CodeGen/PowerPC/ppc-dmf-types.c         | 177 +++++++++++++++++++++
 .../CodeGen/PowerPC/ppc-future-mma-builtin-err.c   |  21 +++
 clang/test/CodeGenCXX/ppc-mangle-mma-types.cpp     |   5 +
 clang/test/Sema/ppc-dmf-types.c                    | 103 ++++++++++++
 .../Plugins/TypeSystem/Clang/TypeSystemClang.cpp   |   1 +
 11 files changed, 445 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/CodeGen/PowerPC/builtins-ppc-dmf.c
 create mode 100644 clang/test/CodeGen/PowerPC/ppc-dmf-paired-vec-memops-builtin-err.c
 create mode 100644 clang/test/CodeGen/PowerPC/ppc-dmf-types.c
 create mode 100644 clang/test/CodeGen/PowerPC/ppc-future-mma-builtin-err.c
 create mode 100644 clang/test/Sema/ppc-dmf-types.c

diff --git a/clang/include/clang/Basic/BuiltinsPPC.def b/clang/include/clang/Basic/BuiltinsPPC.def
index bb7d54bbb793..099500754a0e 100644
--- a/clang/include/clang/Basic/BuiltinsPPC.def
+++ b/clang/include/clang/Basic/BuiltinsPPC.def
@@ -1134,6 +1134,18 @@ UNALIASED_CUSTOM_BUILTIN(mma_pmxvbf16ger2np, "vW512*VVi15i15i3", true,
                          "mma,paired-vector-memops")
 UNALIASED_CUSTOM_BUILTIN(mma_pmxvbf16ger2nn, "vW512*VVi15i15i3", true,
                          "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_dmxvi8gerx4, "vW1024*W256V", false,
+                         "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_pmdmxvi8gerx4, "vW1024*W256Vi255i15i15", false,
+                         "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_dmxvi8gerx4pp, "vW1024*W256V", true,
+                         "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_pmdmxvi8gerx4pp, "vW1024*W256Vi255i15i15", true,
+                         "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_dmxvi8gerx4spp,  "vW1024*W256V", true,
+                         "mma,paired-vector-memops")
+UNALIASED_CUSTOM_BUILTIN(mma_pmdmxvi8gerx4spp, "vW1024*W256Vi255i15i15", true,
+                         "mma,paired-vector-memops")
 
 // FIXME: Obviously incomplete.
 
diff --git a/clang/include/clang/Basic/PPCTypes.def b/clang/include/clang/Basic/PPCTypes.def
index 9e2cb2aedc9f..fc4155ca98b2 100644
--- a/clang/include/clang/Basic/PPCTypes.def
+++ b/clang/include/clang/Basic/PPCTypes.def
@@ -30,6 +30,7 @@
 #endif
 
 
+PPC_VECTOR_MMA_TYPE(__dmr1024, DMR1024, 1024)
 PPC_VECTOR_MMA_TYPE(__vector_quad, VectorQuad, 512)
 PPC_VECTOR_VSX_TYPE(__vector_pair, VectorPair, 256)
 
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index 74be2871f270..02d6570d0ea0 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -3522,6 +3522,7 @@ static void encodeTypeForFunctionPointerAuth(const ASTContext &Ctx,
     case BuiltinType::BFloat16:
     case BuiltinType::VectorQuad:
     case BuiltinType::VectorPair:
+    case BuiltinType::DMR1024:
       OS << "?";
       return;
 
diff --git a/clang/test/AST/ast-dump-ppc-types.c b/clang/test/AST/ast-dump-ppc-types.c
index 26ae5441f20d..1c860c268e0e 100644
--- a/clang/test/AST/ast-dump-ppc-types.c
+++ b/clang/test/AST/ast-dump-ppc-types.c
@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -target-cpu future \
+// RUN:   -ast-dump  %s | FileCheck %s
 // RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -target-cpu pwr10 \
-// RUN:   -ast-dump -ast-dump-filter __vector %s | FileCheck %s
+// RUN:   -ast-dump  %s | FileCheck %s
 // RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -target-cpu pwr9 \
-// RUN:   -ast-dump -ast-dump-filter __vector %s | FileCheck %s
+// RUN:   -ast-dump  %s | FileCheck %s
 // RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -target-cpu pwr8 \
-// RUN:   -ast-dump -ast-dump-filter __vector %s | FileCheck %s
+// RUN:   -ast-dump  %s | FileCheck %s
 // RUN: %clang_cc1 -triple x86_64-unknown-unknown -ast-dump %s | FileCheck %s \
 // RUN:   --check-prefix=CHECK-X86_64
 // RUN: %clang_cc1 -triple arm-unknown-unknown -ast-dump %s | FileCheck %s \
@@ -15,16 +17,21 @@
 // are correctly defined. We also added checks on a couple of other targets to
 // ensure the types are target-dependent.
 
+// CHECK: TypedefDecl {{.*}} implicit __dmr1024 '__dmr1024'
+// CHECK: `-BuiltinType {{.*}} '__dmr1024'
 // CHECK: TypedefDecl {{.*}} implicit __vector_quad '__vector_quad'
 // CHECK-NEXT: -BuiltinType {{.*}} '__vector_quad'
 // CHECK: TypedefDecl {{.*}} implicit __vector_pair '__vector_pair'
 // CHECK-NEXT: -BuiltinType {{.*}} '__vector_pair'
 
+// CHECK-X86_64-NOT: __dmr1024
 // CHECK-X86_64-NOT: __vector_quad
 // CHECK-X86_64-NOT: __vector_pair
 
+// CHECK-ARM-NOT: __dmr1024
 // CHECK-ARM-NOT: __vector_quad
 // CHECK-ARM-NOT: __vector_pair
 
+// CHECK-RISCV64-NOT: __dmr1024
 // CHECK-RISCV64-NOT: __vector_quad
 // CHECK-RISCV64-NOT: __vector_pair
diff --git a/clang/test/CodeGen/PowerPC/builtins-ppc-dmf.c b/clang/test/CodeGen/PowerPC/builtins-ppc-dmf.c
new file mode 100644
index 000000000000..41f13155847b
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/builtins-ppc-dmf.c
@@ -0,0 +1,94 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -O3 -triple powerpc64le-unknown-unknown -target-cpu future \
+// RUN:  -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -O3 -triple powerpc64-ibm-aix -target-cpu future \
+// RUN: -emit-llvm %s -o - | FileCheck %s
+
+
+// CHECK-LABEL: @test_dmxvi8gerx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2:![0-9]+]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.dmxvi8gerx4(<256 x i1> [[TMP0]], <16 x i8> [[VC:%.*]])
+// CHECK-NEXT:    store <1024 x i1> [[TMP1]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6:![0-9]+]]
+// CHECK-NEXT:    ret void
+//
+void test_dmxvi8gerx4(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_dmxvi8gerx4(&vdmr, vp, vc);
+  *((__dmr1024 *)resp) = vdmr;
+}
+
+// CHECK-LABEL: @test_pmdmxvi8gerx4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvi8gerx4(<256 x i1> [[TMP0]], <16 x i8> [[VC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    store <1024 x i1> [[TMP1]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test_pmdmxvi8gerx4(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_pmdmxvi8gerx4(&vdmr, vp, vc, 0, 0, 0);
+  *((__dmr1024 *)resp) = vdmr;
+}
+
+// CHECK-LABEL: @test_dmxvi8gerx4pp(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <1024 x i1>, ptr [[VDMRP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.dmxvi8gerx4pp(<1024 x i1> [[TMP0]], <256 x i1> [[TMP1]], <16 x i8> [[VC:%.*]])
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test_dmxvi8gerx4pp(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_dmxvi8gerx4pp(&vdmr, vp, vc);
+  *((__dmr1024 *)resp) = vdmr;
+}
+
+// CHECK-LABEL: @test_pmdmxvi8gerx4pp(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <1024 x i1>, ptr [[VDMRP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvi8gerx4pp(<1024 x i1> [[TMP0]], <256 x i1> [[TMP1]], <16 x i8> [[VC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test_pmdmxvi8gerx4pp(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_pmdmxvi8gerx4pp(&vdmr, vp, vc, 0, 0, 0);
+  *((__dmr1024 *)resp) = vdmr;
+}
+
+// CHECK-LABEL: @test_dmxvi8gerx4spp(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <1024 x i1>, ptr [[VDMRP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.dmxvi8gerx4spp(<1024 x i1> [[TMP0]], <256 x i1> [[TMP1]], <16 x i8> [[VC:%.*]])
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test_dmxvi8gerx4spp(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_dmxvi8gerx4spp(&vdmr, vp, vc);
+  *((__dmr1024 *)resp) = vdmr;
+}
+
+// CHECK-LABEL: @test_pmdmxvi8gerx4spp(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = load <1024 x i1>, ptr [[VDMRP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    [[TMP1:%.*]] = load <256 x i1>, ptr [[VPP:%.*]], align 32, !tbaa [[TBAA2]]
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <1024 x i1> @llvm.ppc.mma.pmdmxvi8gerx4spp(<1024 x i1> [[TMP0]], <256 x i1> [[TMP1]], <16 x i8> [[VC:%.*]], i32 0, i32 0, i32 0)
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[RESP:%.*]], align 128, !tbaa [[TBAA6]]
+// CHECK-NEXT:    ret void
+//
+void test_pmdmxvi8gerx4spp(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc, unsigned char *resp) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_pmdmxvi8gerx4spp(&vdmr, vp, vc, 0, 0, 0);
+  *((__dmr1024 *)resp) = vdmr;
+}
diff --git a/clang/test/CodeGen/PowerPC/ppc-dmf-paired-vec-memops-builtin-err.c b/clang/test/CodeGen/PowerPC/ppc-dmf-paired-vec-memops-builtin-err.c
new file mode 100644
index 000000000000..eef1abca2241
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/ppc-dmf-paired-vec-memops-builtin-err.c
@@ -0,0 +1,20 @@
+// RUN: not %clang_cc1 -triple powerpc64le-unknown-linux-gnu -target-cpu future \
+// RUN:   %s -emit-llvm-only 2>&1 | FileCheck %s
+
+__attribute__((target("no-paired-vector-memops")))
+void test_pair(unsigned char *vdmr, unsigned char *vpp, vector unsigned char vc) {
+   __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_dmxvi8gerx4((__dmr1024 *)vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4((__dmr1024 *)vdmr, vp, vc, 0, 0, 0);
+  __builtin_mma_dmxvi8gerx4pp((__dmr1024 *)vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4pp((__dmr1024 *)vdmr, vp, vc, 0, 0, 0);
+  __builtin_mma_dmxvi8gerx4spp((__dmr1024 *)vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4spp((__dmr1024 *)vdmr, vp, vc, 0, 0, 0);
+
+// CHECK: error: '__builtin_mma_dmxvi8gerx4' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_dmxvi8gerx4pp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4pp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_dmxvi8gerx4spp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4spp' needs target feature mma,paired-vector-memops
+}
diff --git a/clang/test/CodeGen/PowerPC/ppc-dmf-types.c b/clang/test/CodeGen/PowerPC/ppc-dmf-types.c
new file mode 100644
index 000000000000..9dff289370eb
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/ppc-dmf-types.c
@@ -0,0 +1,177 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -triple powerpc64le-linux-unknown -target-cpu future \
+// RUN:   -emit-llvm -o - %s | FileCheck %s
+
+
+// CHECK-LABEL: @test_dmr_copy(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR1_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PTR2_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[PTR1:%.*]], ptr [[PTR1_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[PTR2:%.*]], ptr [[PTR2_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR1_ADDR]], align 8
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds <1024 x i1>, ptr [[TMP0]], i64 2
+// CHECK-NEXT:    [[TMP1:%.*]] = load <1024 x i1>, ptr [[ADD_PTR]], align 128
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[PTR2_ADDR]], align 8
+// CHECK-NEXT:    [[ADD_PTR1:%.*]] = getelementptr inbounds <1024 x i1>, ptr [[TMP2]], i64 1
+// CHECK-NEXT:    store <1024 x i1> [[TMP1]], ptr [[ADD_PTR1]], align 128
+// CHECK-NEXT:    ret void
+//
+void test_dmr_copy(__dmr1024 *ptr1, __dmr1024 *ptr2) {
+  *(ptr2 + 1) = *(ptr1 + 2);
+}
+
+// CHECK-LABEL: @test_dmr_typedef(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[INP_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[OUTP_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRIN:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMROUT:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[INP:%.*]], ptr [[INP_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[OUTP:%.*]], ptr [[OUTP_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[INP_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRIN]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[OUTP_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr [[VDMROUT]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[VDMRIN]], align 8
+// CHECK-NEXT:    [[TMP3:%.*]] = load <1024 x i1>, ptr [[TMP2]], align 128
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[VDMROUT]], align 8
+// CHECK-NEXT:    store <1024 x i1> [[TMP3]], ptr [[TMP4]], align 128
+// CHECK-NEXT:    ret void
+//
+void test_dmr_typedef(int *inp, int *outp) {
+  __dmr1024 *vdmrin = (__dmr1024 *)inp;
+  __dmr1024 *vdmrout = (__dmr1024 *)outp;
+  *vdmrout = *vdmrin;
+}
+
+// CHECK-LABEL: @test_dmr_arg(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VDMR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[VDMR:%.*]], ptr [[VDMR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <1024 x i1>, ptr [[TMP1]], align 128
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[TMP3]], align 128
+// CHECK-NEXT:    ret void
+//
+void test_dmr_arg(__dmr1024 *vdmr, int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = *vdmr;
+}
+
+// CHECK-LABEL: @test_dmr_const_arg(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VDMR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[VDMR:%.*]], ptr [[VDMR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <1024 x i1>, ptr [[TMP1]], align 128
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[TMP3]], align 128
+// CHECK-NEXT:    ret void
+//
+void test_dmr_const_arg(const __dmr1024 *const vdmr, int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = *vdmr;
+}
+
+// CHECK-LABEL: @test_dmr_array_arg(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[VDMRA_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[VDMRA:%.*]], ptr [[VDMRA_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMRA_ADDR]], align 8
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds <1024 x i1>, ptr [[TMP1]], i64 0
+// CHECK-NEXT:    [[TMP2:%.*]] = load <1024 x i1>, ptr [[ARRAYIDX]], align 128
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[TMP3]], align 128
+// CHECK-NEXT:    ret void
+//
+void test_dmr_array_arg(__dmr1024 vdmra[], int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = vdmra[0];
+}
+
+// CHECK-LABEL: @test_dmr_ret(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds <1024 x i1>, ptr [[TMP1]], i64 2
+// CHECK-NEXT:    ret ptr [[ADD_PTR]]
+//
+__dmr1024 *test_dmr_ret(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  return vdmrp + 2;
+}
+
+// CHECK-LABEL: @test_dmr_ret_const(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds <1024 x i1>, ptr [[TMP1]], i64 2
+// CHECK-NEXT:    ret ptr [[ADD_PTR]]
+//
+const __dmr1024 *test_dmr_ret_const(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  return vdmrp + 2;
+}
+
+// CHECK-LABEL: @test_dmr_sizeof_alignof(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[PTR_ADDR:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMRP:%.*]] = alloca ptr, align 8
+// CHECK-NEXT:    [[VDMR:%.*]] = alloca <1024 x i1>, align 128
+// CHECK-NEXT:    [[SIZET:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[ALIGNT:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[SIZEV:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[ALIGNV:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    store ptr [[PTR:%.*]], ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[PTR_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP0]], ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[VDMRP]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load <1024 x i1>, ptr [[TMP1]], align 128
+// CHECK-NEXT:    store <1024 x i1> [[TMP2]], ptr [[VDMR]], align 128
+// CHECK-NEXT:    store i32 128, ptr [[SIZET]], align 4
+// CHECK-NEXT:    store i32 128, ptr [[ALIGNT]], align 4
+// CHECK-NEXT:    store i32 128, ptr [[SIZEV]], align 4
+// CHECK-NEXT:    store i32 128, ptr [[ALIGNV]], align 4
+// CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[SIZET]], align 4
+// CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ALIGNT]], align 4
+// CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP3]], [[TMP4]]
+// CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[SIZEV]], align 4
+// CHECK-NEXT:    [[ADD1:%.*]] = add i32 [[ADD]], [[TMP5]]
+// CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ALIGNV]], align 4
+// CHECK-NEXT:    [[ADD2:%.*]] = add i32 [[ADD1]], [[TMP6]]
+// CHECK-NEXT:    ret i32 [[ADD2]]
+//
+int test_dmr_sizeof_alignof(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  __dmr1024 vdmr = *vdmrp;
+  unsigned sizet = sizeof(__dmr1024);
+  unsigned alignt = __alignof__(__dmr1024);
+   unsigned sizev = sizeof(vdmr);
+  unsigned alignv = __alignof__(vdmr);
+  return sizet + alignt + sizev + alignv;
+}
diff --git a/clang/test/CodeGen/PowerPC/ppc-future-mma-builtin-err.c b/clang/test/CodeGen/PowerPC/ppc-future-mma-builtin-err.c
new file mode 100644
index 000000000000..1b8d345ac7ec
--- /dev/null
+++ b/clang/test/CodeGen/PowerPC/ppc-future-mma-builtin-err.c
@@ -0,0 +1,21 @@
+// RUN: not %clang_cc1 -triple powerpc64le-unknown-linux-gnu -target-cpu future \
+// RUN:   %s -emit-llvm-only 2>&1 | FileCheck %s
+
+__attribute__((target("no-mma")))
+void test_mma(unsigned char *vdmrp, unsigned char *vpp, vector unsigned char vc) {
+  __dmr1024 vdmr = *((__dmr1024 *)vdmrp);
+  __vector_pair vp = *((__vector_pair *)vpp);
+  __builtin_mma_dmxvi8gerx4(&vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4(&vdmr, vp, vc, 0, 0, 0);
+  __builtin_mma_dmxvi8gerx4pp(&vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4pp(&vdmr, vp, vc, 0, 0, 0);
+  __builtin_mma_dmxvi8gerx4spp(&vdmr, vp, vc);
+  __builtin_mma_pmdmxvi8gerx4spp(&vdmr, vp, vc, 0, 0, 0);
+
+// CHECK: error: '__builtin_mma_dmxvi8gerx4' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_dmxvi8gerx4pp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4pp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_dmxvi8gerx4spp' needs target feature mma,paired-vector-memops
+// CHECK: error: '__builtin_mma_pmdmxvi8gerx4spp' needs target feature mma,paired-vector-memops
+}
diff --git a/clang/test/CodeGenCXX/ppc-mangle-mma-types.cpp b/clang/test/CodeGenCXX/ppc-mangle-mma-types.cpp
index 74e50ceea386..1e213e7f7512 100644
--- a/clang/test/CodeGenCXX/ppc-mangle-mma-types.cpp
+++ b/clang/test/CodeGenCXX/ppc-mangle-mma-types.cpp
@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -triple powerpc64le-linux-unknown -target-cpu future %s \
+// RUN:   -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple powerpc64le-linux-unknown -target-cpu pwr10 %s \
 // RUN:   -emit-llvm -o - | FileCheck %s
 // RUN: %clang_cc1 -triple powerpc64le-linux-unknown -target-cpu pwr9 %s \
@@ -5,6 +7,9 @@
 // RUN: %clang_cc1 -triple powerpc64le-linux-unknown -target-cpu pwr8 %s \
 // RUN:   -emit-llvm -o - | FileCheck %s
 
+// CHECK: _Z2f0Pu9__dmr1024
+void f0(__dmr1024 *vdmr) {}
+
 // CHECK: _Z2f1Pu13__vector_quad
 void f1(__vector_quad *vq) {}
 
diff --git a/clang/test/Sema/ppc-dmf-types.c b/clang/test/Sema/ppc-dmf-types.c
new file mode 100644
index 000000000000..b3da72df2508
--- /dev/null
+++ b/clang/test/Sema/ppc-dmf-types.c
@@ -0,0 +1,103 @@
+// RUN: %clang_cc1 -triple powerpc64le-unknown-unknown -fsyntax-only \
+// RUN:   -target-cpu future %s -verify
+// RUN: %clang_cc1 -triple powerpc64-unknown-unknown -fsyntax-only \
+// RUN:   -target-cpu future %s -verify
+
+// The use of PPC MMA types is strongly restricted. Non-pointer MMA variables
+// can only be declared in functions and a limited number of operations are
+// supported on these types. This test case checks that invalid uses of MMA
+// types are correctly prevented.
+
+// vector dmr
+
+// typedef
+typedef __dmr1024 dmr_t;
+
+// function argument
+void testDmrArg1(__dmr1024 vdmr, int *ptr) { // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = vdmr;
+}
+
+void testDmrArg2(const __dmr1024 vdmr, int *ptr) { // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = vdmr;
+}
+
+void testDmrArg3(const dmr_t vdmr, int *ptr) { // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  *vdmrp = vdmr;
+}
+
+// function return
+__dmr1024 testDmrRet1(int *ptr) { // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  return *vdmrp; // expected-error {{invalid use of PPC MMA type}}
+}
+
+const dmr_t testDmrRet4(int *ptr) { // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  return *vdmrp; // expected-error {{invalid use of PPC MMA type}}
+}
+
+// global
+__dmr1024 globalvdmr;        // expected-error {{invalid use of PPC MMA type}}
+const __dmr1024 globalvdmr2; // expected-error {{invalid use of PPC MMA type}}
+__dmr1024 *globalvdmrp;
+const __dmr1024 *const globalvdmrp2;
+dmr_t globalvdmr_t; // expected-error {{invalid use of PPC MMA type}}
+
+// struct field
+struct TestDmrStruct {
+  int a;
+  float b;
+  __dmr1024 c; // expected-error {{invalid use of PPC MMA type}}
+  __dmr1024 *vq;
+};
+
+// operators
+int testDmrOperators1(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  __dmr1024 vdmr1 = *(vdmrp + 0);
+  __dmr1024 vdmr2 = *(vdmrp + 1);
+  __dmr1024 vdmr3 = *(vdmrp + 2);
+  if (vdmr1) // expected-error {{statement requires expression of scalar type ('__dmr1024' invalid)}}
+    *(vdmrp + 10) = vdmr1;
+  if (!vdmr2) // expected-error {{invalid argument type '__dmr1024' to unary expression}}
+    *(vdmrp + 11) = vdmr3;
+  int c1 = vdmr1 && vdmr2; // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  int c2 = vdmr2 == vdmr3; // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  int c3 = vdmr2 < vdmr1;  // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  return c1 || c2 || c3;
+}
+
+void testDmrOperators2(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  __dmr1024 vdmr1 = *(vdmrp + 0);
+  __dmr1024 vdmr2 = *(vdmrp + 1);
+  __dmr1024 vdmr3 = *(vdmrp + 2);
+  vdmr1 = -vdmr1;        // expected-error {{invalid argument type '__dmr1024' to unary expression}}
+  vdmr2 = vdmr1 + vdmr3; // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  vdmr2 = vdmr2 * vdmr3; // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  vdmr3 = vdmr3 | vdmr3; // expected-error {{invalid operands to binary expression ('__dmr1024' and '__dmr1024')}}
+  vdmr3 = vdmr3 << 2;    // expected-error {{invalid operands to binary expression ('__dmr1024' and 'int')}}
+  *(vdmrp + 10) = vdmr1;
+  *(vdmrp + 11) = vdmr2;
+  *(vdmrp + 12) = vdmr3;
+}
+
+vector unsigned char testDmrOperators3(int *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  __dmr1024 vdmr1 = *(vdmrp + 0);
+  __dmr1024 vdmr2 = *(vdmrp + 1);
+  __dmr1024 vdmr3 = *(vdmrp + 2);
+  vdmr1 ? *(vdmrp + 10) = vdmr2 : *(vdmrp + 11) = vdmr3; // expected-error {{used type '__dmr1024' where arithmetic or pointer type is required}}
+  vdmr2 = vdmr3;
+  return vdmr2[1]; // expected-error {{subscripted value is not an array, pointer, or vector}}
+}
+
+void testDmrOperators4(int v, void *ptr) {
+  __dmr1024 *vdmrp = (__dmr1024 *)ptr;
+  __dmr1024 vdmr1 = (__dmr1024)v;   // expected-error {{used type '__dmr1024' where arithmetic or pointer type is required}}
+  __dmr1024 vdmr2 = (__dmr1024)vdmrp; // expected-error {{used type '__dmr1024' where arithmetic or pointer type is required}}
+}
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 29302413cf8f..82e07bb8e0ff 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -5024,6 +5024,7 @@ lldb::Encoding TypeSystemClang::GetEncoding(lldb::opaque_compiler_type_t type,
     // PowerPC -- Matrix Multiply Assist
     case clang::BuiltinType::VectorPair:
     case clang::BuiltinType::VectorQuad:
+    case clang::BuiltinType::DMR1024:
       break;
 
     // ARM -- Scalable Vector Extension
-- 
cgit v1.2.3


From f8ee5774b612c425051adfc9108d1c52820b193d Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 18:09:07 +0100
Subject: [X86] combineConcatVectorOps - only concat AVX1 v4i64 shift-by-32 to
 a shuffle if the concat is free (#145043)

---
 llvm/lib/Target/X86/X86ISelLowering.cpp        | 20 +++++++++++---------
 llvm/test/CodeGen/X86/vector-shift-lshr-256.ll | 18 ++++++------------
 llvm/test/CodeGen/X86/vector-shift-shl-256.ll  | 18 ++++++------------
 3 files changed, 23 insertions(+), 33 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index c32737010560..35d7b8084405 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -58844,16 +58844,18 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
           llvm::all_of(Ops, [](SDValue Op) {
             return Op.getConstantOperandAPInt(1) == 32;
           })) {
-        SDValue Res = DAG.getBitcast(MVT::v8i32, ConcatSubOperand(VT, Ops, 0));
-        SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
-        if (Opcode == X86ISD::VSHLI) {
-          Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
-                                     {8, 0, 8, 2, 8, 4, 8, 6});
-        } else {
-          Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
-                                     {1, 8, 3, 8, 5, 8, 7, 8});
+        if (SDValue Res = CombineSubOperand(VT, Ops, 0)) {
+          SDValue Zero = getZeroVector(MVT::v8i32, Subtarget, DAG, DL);
+          Res = DAG.getBitcast(MVT::v8i32, Res);
+          if (Opcode == X86ISD::VSHLI) {
+            Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
+                                       {8, 0, 8, 2, 8, 4, 8, 6});
+          } else {
+            Res = DAG.getVectorShuffle(MVT::v8i32, DL, Res, Zero,
+                                       {1, 8, 3, 8, 5, 8, 7, 8});
+          }
+          return DAG.getBitcast(VT, Res);
         }
-        return DAG.getBitcast(VT, Res);
       }
       [[fallthrough]];
     case X86ISD::VSRAI:
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
index b45525b6e20f..3a4bb223618d 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll
@@ -1974,11 +1974,9 @@ define <4 x i64> @shift32_v4i64(<4 x i64> %a) nounwind {
 define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ; AVX1-LABEL: shift32_v4i64_concat:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shift32_v4i64_concat:
@@ -1990,11 +1988,9 @@ define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ;
 ; XOPAVX1-LABEL: shift32_v4i64_concat:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsrlq $32, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shift32_v4i64_concat:
@@ -2020,11 +2016,9 @@ define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ;
 ; X86-AVX1-LABEL: shift32_v4i64_concat:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsrlq $32, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[1,3],ymm1[1,3],ymm0[5,7],ymm1[5,7]
-; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: shift32_v4i64_concat:
diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
index 2248ee997d52..b56a8b5b2294 100644
--- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll
@@ -1827,11 +1827,9 @@ define <4 x i64> @shift32_v4i64(<4 x i64> %a) nounwind {
 define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ; AVX1-LABEL: shift32_v4i64_concat:
 ; AVX1:       # %bb.0:
-; AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; AVX1-NEXT:    retq
 ;
 ; AVX2-LABEL: shift32_v4i64_concat:
@@ -1843,11 +1841,9 @@ define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ;
 ; XOPAVX1-LABEL: shift32_v4i64_concat:
 ; XOPAVX1:       # %bb.0:
-; XOPAVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; XOPAVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; XOPAVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; XOPAVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; XOPAVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; XOPAVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; XOPAVX1-NEXT:    retq
 ;
 ; XOPAVX2-LABEL: shift32_v4i64_concat:
@@ -1873,11 +1869,9 @@ define <4 x i64> @shift32_v4i64_concat(<2 x i64> %lo, <2 x i64> %hi) nounwind {
 ;
 ; X86-AVX1-LABEL: shift32_v4i64_concat:
 ; X86-AVX1:       # %bb.0:
-; X86-AVX1-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX1-NEXT:    vpsllq $32, %xmm0, %xmm0
+; X86-AVX1-NEXT:    vpsllq $32, %xmm1, %xmm1
 ; X86-AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; X86-AVX1-NEXT:    vxorps %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm1[0,2],ymm0[0,2],ymm1[4,6],ymm0[4,6]
-; X86-AVX1-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
 ; X86-AVX1-NEXT:    retl
 ;
 ; X86-AVX2-LABEL: shift32_v4i64_concat:
-- 
cgit v1.2.3


From 33a92af1b2260506356eb838125b356703bf02bb Mon Sep 17 00:00:00 2001
From: Thurston Dang <thurston@google.com>
Date: Fri, 20 Jun 2025 10:11:12 -0700
Subject: [msan] Add off-by-default flag to fix false negatives from partially
 undefined constant fixed-length vectors (#143837)

This patch adds an off-by-default flag which, when enabled via `-mllvm -msan-poison-undef-vectors=true`, fixes a false negative in MSan (partially-undefined constant fixed-length vectors). It is currently off by default since, by fixing the false positive, code/tests that previously passed MSan may start failing. The default will be changed in a future patch.

Prior to this patch, MSan computes that partially-undefined constant fixed-length vectors are fully initialized, which leads to false negatives; moreover, benign vector rewriting could theoretically flip MSan's shadow computation from initialized to uninitialized or vice-versa (*). `-msan-poison-undef-vectors=true` calculates the shadow precisely: for each element of the vector, the corresponding shadow is fully uninitialized if the element is undefined/poisoned, otherwise it is fully initialized.

Updates the test from https://github.com/llvm/llvm-project/pull/143823

(*) For example:
  ```
  %x = insertelement <2 x i64> <i64 0, i64 poison>, i64 42, i64 0
  %y = insertelement <2 x i64> <i64 poison, i64 poison>, i64 42, i64 0
  ```
%x and %y are equivalent but, prior to this patch, MSan incorrectly computes the shadow of %x as <0, 0> rather than <0, -1>.
---
 .../Transforms/Instrumentation/MemorySanitizer.cpp | 46 +++++++++++++++++++---
 .../MemorySanitizer/partial-poison.ll              | 39 ++++++++++++++----
 2 files changed, 73 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
index fb55bd7bfe56..5aeca60b08d3 100644
--- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
+++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp
@@ -265,9 +265,22 @@ static cl::opt<bool>
                       cl::desc("Print name of local stack variable"),
                       cl::Hidden, cl::init(true));
 
-static cl::opt<bool> ClPoisonUndef("msan-poison-undef",
-                                   cl::desc("poison undef temps"), cl::Hidden,
-                                   cl::init(true));
+static cl::opt<bool>
+    ClPoisonUndef("msan-poison-undef",
+                  cl::desc("Poison fully undef temporary values. "
+                           "Partially undefined constant vectors "
+                           "are unaffected by this flag (see "
+                           "-msan-poison-undef-vectors)."),
+                  cl::Hidden, cl::init(true));
+
+static cl::opt<bool> ClPoisonUndefVectors(
+    "msan-poison-undef-vectors",
+    cl::desc("Precisely poison partially undefined constant vectors. "
+             "If false (legacy behavior), the entire vector is "
+             "considered fully initialized, which may lead to false "
+             "negatives. Fully undefined constant vectors are "
+             "unaffected by this flag (see -msan-poison-undef)."),
+    cl::Hidden, cl::init(false));
 
 static cl::opt<bool>
     ClHandleICmp("msan-handle-icmp",
@@ -1181,6 +1194,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
   bool PropagateShadow;
   bool PoisonStack;
   bool PoisonUndef;
+  bool PoisonUndefVectors;
 
   struct ShadowOriginAndInsertPoint {
     Value *Shadow;
@@ -1207,6 +1221,7 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
     PropagateShadow = SanitizeFunction;
     PoisonStack = SanitizeFunction && ClPoisonStack;
     PoisonUndef = SanitizeFunction && ClPoisonUndef;
+    PoisonUndefVectors = SanitizeFunction && ClPoisonUndefVectors;
 
     // In the presence of unreachable blocks, we may see Phi nodes with
     // incoming nodes from such blocks. Since InstVisitor skips unreachable
@@ -1989,6 +2004,8 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       }
       return Shadow;
     }
+    // Handle fully undefined values
+    // (partially undefined constant vectors are handled later)
     if (UndefValue *U = dyn_cast<UndefValue>(V)) {
       Value *AllOnes = (PropagateShadow && PoisonUndef) ? getPoisonedShadow(V)
                                                         : getCleanShadow(V);
@@ -2086,8 +2103,27 @@ struct MemorySanitizerVisitor : public InstVisitor<MemorySanitizerVisitor> {
       return ShadowPtr;
     }
 
-    // TODO: Partially undefined vectors are handled by the fall-through case
-    //       below (see partial-poison.ll); this causes false negatives.
+    // Check for partially-undefined constant vectors
+    // TODO: scalable vectors (this is hard because we do not have IRBuilder)
+    if (isa<FixedVectorType>(V->getType()) && isa<Constant>(V) &&
+        cast<Constant>(V)->containsUndefOrPoisonElement() && PropagateShadow &&
+        PoisonUndefVectors) {
+      unsigned NumElems = cast<FixedVectorType>(V->getType())->getNumElements();
+      SmallVector<Constant *, 32> ShadowVector(NumElems);
+      for (unsigned i = 0; i != NumElems; ++i) {
+        Constant *Elem = cast<Constant>(V)->getAggregateElement(i);
+        ShadowVector[i] = isa<UndefValue>(Elem) ? getPoisonedShadow(Elem)
+                                                : getCleanShadow(Elem);
+      }
+
+      Value *ShadowConstant = ConstantVector::get(ShadowVector);
+      LLVM_DEBUG(dbgs() << "Partial undef constant vector: " << *V << " ==> "
+                        << *ShadowConstant << "\n");
+
+      return ShadowConstant;
+    }
+
+    // TODO: partially-undefined constant arrays, structures, and nested types
 
     // For everything else the shadow is zero.
     return getCleanShadow(V);
diff --git a/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll b/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll
index 5164441c17e1..025317a53c8d 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/partial-poison.ll
@@ -1,8 +1,9 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt < %s -S -passes='msan' 2>&1 | FileCheck %s
+; RUN: opt < %s -S -passes='msan' -msan-poison-undef-vectors=true  2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-PRECISE
+; RUN: opt < %s -S -passes='msan' -msan-poison-undef-vectors=false 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-IMPRECISE
 ;
-; Test case to show that MSan computes shadows for partially poisoned vectors
-; as fully initialized, resulting in false negatives.
+; Regression test case for computing shadows of partially poisoned vectors.
+; Partially poisoned structs and arrays are not correctly implemented.
 
 target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
@@ -11,7 +12,8 @@ define <2 x i64> @left_poison(ptr %add.ptr) sanitize_memory {
 ; CHECK-LABEL: define <2 x i64> @left_poison(
 ; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-PRECISE:   store <2 x i64> <i64 -1, i64 0>, ptr @__msan_retval_tls, align 8
+; CHECK-IMPRECISE: store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> <i64 poison, i64 42>
 ;
   ret <2 x i64> <i64 poison, i64 42>
@@ -21,7 +23,8 @@ define <2 x i64> @right_poison(ptr %add.ptr) sanitize_memory {
 ; CHECK-LABEL: define <2 x i64> @right_poison(
 ; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-PRECISE:   store <2 x i64> <i64 0, i64 -1>, ptr @__msan_retval_tls, align 8
+; CHECK-IMPRECISE: store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> <i64 42, i64 poison>
 ;
   ret <2 x i64> <i64 42, i64 poison>
@@ -51,7 +54,8 @@ define <2 x i64> @left_undef(ptr %add.ptr) sanitize_memory {
 ; CHECK-LABEL: define <2 x i64> @left_undef(
 ; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-PRECISE:   store <2 x i64> <i64 -1, i64 0>, ptr @__msan_retval_tls, align 8
+; CHECK-IMPRECISE: store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> <i64 undef, i64 42>
 ;
   ret <2 x i64> <i64 undef, i64 42>
@@ -61,7 +65,8 @@ define <2 x i64> @right_undef(ptr %add.ptr) sanitize_memory {
 ; CHECK-LABEL: define <2 x i64> @right_undef(
 ; CHECK-SAME: ptr [[ADD_PTR:%.*]]) #[[ATTR0]] {
 ; CHECK-NEXT:    call void @llvm.donothing()
-; CHECK-NEXT:    store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-PRECISE:   store <2 x i64> <i64 0, i64 -1>, ptr @__msan_retval_tls, align 8
+; CHECK-IMPRECISE: store <2 x i64> zeroinitializer, ptr @__msan_retval_tls, align 8
 ; CHECK-NEXT:    ret <2 x i64> <i64 42, i64 undef>
 ;
   ret <2 x i64> <i64 42, i64 undef>
@@ -76,3 +81,23 @@ define <2 x i64> @full_undef(ptr %add.ptr) sanitize_memory {
 ;
   ret <2 x i64> <i64 undef, i64 undef>
 }
+
+define {i64, i64}  @struct_left_undef() sanitize_memory {
+; CHECK-LABEL: define { i64, i64 } @struct_left_undef(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store { i64, i64 } zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret { i64, i64 } { i64 undef, i64 42 }
+;
+  ret {i64, i64} { i64 undef, i64 42 }
+}
+
+define [2x i64]  @array_right_undef() sanitize_memory {
+; CHECK-LABEL: define [2 x i64] @array_right_undef(
+; CHECK-SAME: ) #[[ATTR0]] {
+; CHECK-NEXT:    call void @llvm.donothing()
+; CHECK-NEXT:    store [2 x i64] zeroinitializer, ptr @__msan_retval_tls, align 8
+; CHECK-NEXT:    ret [2 x i64] [i64 42, i64 undef]
+; 
+  ret [2x i64] [ i64 42, i64 undef ]
+}
-- 
cgit v1.2.3


From 0d21c956a5c1640c0f9588b307963bf32b09f6ab Mon Sep 17 00:00:00 2001
From: Michal Rostecki <vadorovsky@disroot.org>
Date: Fri, 20 Jun 2025 19:17:36 +0200
Subject: [BPF] Handle nested wrapper structs in BPF map definition traversal
 (#144097)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

In Aya/Rust, BPF map definitions are nested in two nested types:

* A struct representing the map type (e.g., `HashMap`, `RingBuf`) that
provides methods for interacting with the map type (e.g. `HashMap::get`,
`RingBuf::reserve`).
* An `UnsafeCell`, which informs the Rust compiler that the type is
thread-safe and can be safely mutated even as a global variable. The
kernel guarantees map operation safety.

This leads to a type hierarchy like:

```rust
    pub struct HashMap<K, V, const M: usize, const F: usize = 0>(
        core::cell::UnsafeCell<HashMapDef<K, V, M, F>>,
    );
    const BPF_MAP_TYPE_HASH: usize = 1;
    pub struct HashMapDef<K, V, const M: usize, const F: usize = 0> {
        r#type: *const [i32; BPF_MAP_TYPE_HASH],
        key: *const K,
        value: *const V,
        max_entries: *const [i32; M],
        map_flags: *const [i32; F],
    }
```

Then used in the BPF program code as a global variable:

```rust
    #[link_section = ".maps"]
    static HASH_MAP: HashMap<u32, u32, 1337> = HashMap::new();
```

Which is an equivalent of the following BPF map definition in C:

```c
    #define BPF_MAP_TYPE_HASH 1
    struct {
        int (*type)[BPF_MAP_TYPE_HASH];
        typeof(int) *key;
        typeof(int) *value;
        int (*max_entries)[1337];
    } map_1 __attribute__((section(".maps")));
```

Accessing the actual map definition requires traversing:

```
  HASH_MAP -> __0 -> value
```

Previously, the BPF backend only visited the pointee types of the
outermost struct, and didn’t descend into inner wrappers. This caused
issues when the key/value types were custom structs:

```rust
    // Define custom structs for key and values.
    pub struct MyKey(u32);
    pub struct MyValue(u32);

    #[link_section = ".maps"]
    #[export_name = "HASH_MAP"]
    pub static HASH_MAP: HashMap<MyKey, MyValue, 10> = HashMap::new();
```

These types weren’t fully visited and appeared in BTF as forward
declarations:

```
    #30: <FWD> 'MyKey' kind:struct
    #31: <FWD> 'MyValue' kind:struct
```

The fix is to enhance `visitMapDefType` to recursively visit inner
composite members. If a member is a composite type (likely a wrapper),
it is now also visited using `visitMapDefType`, ensuring that the
pointee types of the innermost stuct members, like `MyKey` and
`MyValue`, are fully resolved in BTF.

With this fix, the correct BTF entries are emitted:

```
    #6: <STRUCT> 'MyKey' sz:4 n:1
            #00 '__0' off:0 --> [7]
    #7: <INT> 'u32' bits:32 off:0
    #8: <PTR> --> [9]
    #9: <STRUCT> 'MyValue' sz:4 n:1
            #00 '__0' off:0 --> [7]
```

Fixes: #143361
---
 llvm/lib/Target/BPF/BTFDebug.cpp            |  17 +++-
 llvm/test/CodeGen/BPF/BTF/map-def-nested.ll | 117 ++++++++++++++++++++++++++++
 2 files changed, 132 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/BPF/BTF/map-def-nested.ll

diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index 1d7c6fc799ff..1e29a0f1e85a 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -976,11 +976,24 @@ void BTFDebug::visitMapDefType(const DIType *Ty, uint32_t &TypeId) {
   if (Tag != dwarf::DW_TAG_structure_type || CTy->isForwardDecl())
     return;
 
-  // Visit all struct members to ensure pointee type is visited
+  // Visit all struct members to ensure their types are visited.
   const DINodeArray Elements = CTy->getElements();
   for (const auto *Element : Elements) {
     const auto *MemberType = cast<DIDerivedType>(Element);
-    visitTypeEntry(MemberType->getBaseType());
+    const DIType *MemberBaseType = MemberType->getBaseType();
+
+    // If the member is a composite type, that may indicate the currently
+    // visited composite type is a wrapper, and the member represents the
+    // actual map definition.
+    // In that case, visit the member with `visitMapDefType` instead of
+    // `visitTypeEntry`, treating it specifically as a map definition rather
+    // than as a regular composite type.
+    const auto *MemberCTy = dyn_cast<DICompositeType>(MemberBaseType);
+    if (MemberCTy) {
+      visitMapDefType(MemberBaseType, TypeId);
+    } else {
+      visitTypeEntry(MemberBaseType);
+    }
   }
 
   // Visit this type, struct or a const/typedef/volatile/restrict type
diff --git a/llvm/test/CodeGen/BPF/BTF/map-def-nested.ll b/llvm/test/CodeGen/BPF/BTF/map-def-nested.ll
new file mode 100644
index 000000000000..ac04be371529
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/BTF/map-def-nested.ll
@@ -0,0 +1,117 @@
+; RUN: llc -mtriple=bpfel -mcpu=v3 -filetype=obj -o %t1 %s
+; RUN: llvm-objcopy --dump-section='.BTF'=%t2 %t1
+; RUN: %python %p/print_btf.py %t2 | FileCheck -check-prefixes=CHECK-BTF-SHORT %s
+; RUN: %python %p/print_btf.py %t2 | FileCheck -check-prefixes=CHECK-BTF %s
+
+; Source code:
+;   struct key { int i; };
+;   struct val { int j; };
+;   
+;   #define __uint(name, val) int (*name)[val]
+;   #define __type(name, val) typeof(val) *name
+;   
+;   struct {
+;      struct {
+;           __uint(type, 1);
+;           __uint(max_entries, 1337);
+;           __type(key, struct key);
+;           __type(value, struct val);
+;       } map_def;
+;   } map __attribute__((section(".maps")));
+; Compilation flag:
+;   clang -target bpf -O2 -g -S -emit-llvm t.c
+
+; ModuleID = 'bpf.c'
+source_filename = "bpf.c"
+target datalayout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128"
+target triple = "bpf"
+
+%struct.anon = type { %struct.anon.0 }
+%struct.anon.0 = type { ptr, ptr, ptr, ptr }
+
+@map = dso_local local_unnamed_addr global %struct.anon zeroinitializer, section ".maps", align 8, !dbg !0
+
+; We expect exactly 4 structs:
+; * key
+; * val
+; * inner map type (the actual definition)
+; * outer map type (the wrapper)
+;
+; CHECK-BTF-SHORT-COUNT-4: STRUCT
+; CHECK-BTF-SHORT-NOT:     STRUCT
+
+; We expect no forward declarations.
+;
+; CHECK-BTF-SHORT-NOT: FWD
+
+; Assert the whole BTF.
+;
+; CHECK-BTF:      [1] PTR '(anon)' type_id=3
+; CHECK-BTF-NEXT: [2] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED
+; CHECK-BTF-NEXT: [3] ARRAY '(anon)' type_id=2 index_type_id=4 nr_elems=1
+; CHECK-BTF-NEXT: [4] INT '__ARRAY_SIZE_TYPE__' size=4 bits_offset=0 nr_bits=32 encoding=(none)
+; CHECK-BTF-NEXT: [5] PTR '(anon)' type_id=6
+; CHECK-BTF-NEXT: [6] ARRAY '(anon)' type_id=2 index_type_id=4 nr_elems=1337
+; CHECK-BTF-NEXT: [7] PTR '(anon)' type_id=8
+;
+; Before bug https://github.com/llvm/llvm-project/issues/143361 was fixed, the
+; BTF kind of MyKey (#6) and MyValue (#9) would be FWD instead of STRUCT. The
+; main goal of this test is making sure that the full STRUCT BTF is generated
+; for these types.
+;
+; CHECK-BTF-NEXT: [8] STRUCT 'key' size=4 vlen=1
+; CHECK-BTF-NEXT:         'i' type_id=2 bits_offset=0
+; CHECK-BTF-NEXT: [9] PTR '(anon)' type_id=10
+; CHECK-BTF-NEXT: [10] STRUCT 'val' size=4 vlen=1
+; CHECK-BTF-NEXT:         'j' type_id=2 bits_offset=0
+; CHECK-BTF-NEXT: [11] STRUCT '(anon)' size=32 vlen=4
+; CHECK-BTF-NEXT:         'type' type_id=1 bits_offset=0
+; CHECK-BTF-NEXT:         'max_entries' type_id=5 bits_offset=64
+; CHECK-BTF-NEXT:         'key' type_id=7 bits_offset=128
+; CHECK-BTF-NEXT:         'value' type_id=9 bits_offset=192
+; CHECK-BTF-NEXT: [12] STRUCT '(anon)' size=32 vlen=1
+; CHECK-BTF-NEXT:         'map_def' type_id=11 bits_offset=0
+; CHECK-BTF-NEXT: [13] VAR 'map' type_id=12, linkage=global
+; CHECK-BTF-NEXT: [14] DATASEC '.maps' size=0 vlen=1
+; CHECK-BTF-NEXT:         type_id=13 offset=0 size=32
+
+!llvm.dbg.cu = !{!2}
+!llvm.module.flags = !{!31, !32, !33, !34}
+!llvm.ident = !{!35}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(name: "map", scope: !2, file: !3, line: 14, type: !5, isLocal: false, isDefinition: true)
+!2 = distinct !DICompileUnit(language: DW_LANG_C11, file: !3, producer: "clang version 21.0.0git (git@github.com:llvm/llvm-project.git c935bd3798b39330aab2c9ca29a519457d5e5245)", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, globals: !4, splitDebugInlining: false, nameTableKind: None)
+!3 = !DIFile(filename: "bpf.c", directory: "/tmp", checksumkind: CSK_MD5, checksum: "2330cce6d83c72ef5335abc3016de28e")
+!4 = !{!0}
+!5 = distinct !DICompositeType(tag: DW_TAG_structure_type, file: !3, line: 7, size: 256, elements: !6)
+!6 = !{!7}
+!7 = !DIDerivedType(tag: DW_TAG_member, name: "map_def", scope: !5, file: !3, line: 13, baseType: !8, size: 256)
+!8 = distinct !DICompositeType(tag: DW_TAG_structure_type, scope: !5, file: !3, line: 8, size: 256, elements: !9)
+!9 = !{!10, !16, !21, !26}
+!10 = !DIDerivedType(tag: DW_TAG_member, name: "type", scope: !8, file: !3, line: 9, baseType: !11, size: 64)
+!11 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !12, size: 64)
+!12 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 32, elements: !14)
+!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!14 = !{!15}
+!15 = !DISubrange(count: 1)
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "max_entries", scope: !8, file: !3, line: 10, baseType: !17, size: 64, offset: 64)
+!17 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !18, size: 64)
+!18 = !DICompositeType(tag: DW_TAG_array_type, baseType: !13, size: 42784, elements: !19)
+!19 = !{!20}
+!20 = !DISubrange(count: 1337)
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "key", scope: !8, file: !3, line: 11, baseType: !22, size: 64, offset: 128)
+!22 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !23, size: 64)
+!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "key", file: !3, line: 1, size: 32, elements: !24)
+!24 = !{!25}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !23, file: !3, line: 1, baseType: !13, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_member, name: "value", scope: !8, file: !3, line: 12, baseType: !27, size: 64, offset: 192)
+!27 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !28, size: 64)
+!28 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "val", file: !3, line: 2, size: 32, elements: !29)
+!29 = !{!30}
+!30 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !28, file: !3, line: 2, baseType: !13, size: 32)
+!31 = !{i32 7, !"Dwarf Version", i32 5}
+!32 = !{i32 2, !"Debug Info Version", i32 3}
+!33 = !{i32 1, !"wchar_size", i32 4}
+!34 = !{i32 7, !"frame-pointer", i32 2}
+!35 = !{!"clang version 21.0.0git (git@github.com:llvm/llvm-project.git c935bd3798b39330aab2c9ca29a519457d5e5245)"}
-- 
cgit v1.2.3


From adc6228ea07eba401481e218c3e0536a4aa6b8ec Mon Sep 17 00:00:00 2001
From: Charitha Saumya <136391709+charithaintc@users.noreply.github.com>
Date: Fri, 20 Jun 2025 10:43:19 -0700
Subject: [mlir][xegpu] Refine layout assignment in XeGPU SIMT distribution.
 (#142687)

Changes:
* Decouple layout propagation from subgroup distribution and move it to
an independent pass.
* Refine layout assignment to handle control-flow ops correctly (scf.for, scf.while).
* Refine test cases.
---
 .../mlir/Dialect/XeGPU/Transforms/Passes.td        |  19 +-
 mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h |  14 +
 mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt   |   1 +
 .../XeGPU/Transforms/XeGPUPropagateLayout.cpp      | 889 ++++++++++++++++++++
 .../XeGPU/Transforms/XeGPUSubgroupDistribute.cpp   | 923 +++------------------
 mlir/test/Dialect/XeGPU/propagate-layout.mlir      | 430 ++++++++++
 mlir/test/Dialect/XeGPU/subgroup-distribute.mlir   | 280 +++++++
 mlir/test/Dialect/XeGPU/subgroup-distribution.mlir | 275 ------
 .../Dialect/XeGPU/subgroup-map-propagation.mlir    | 622 --------------
 9 files changed, 1726 insertions(+), 1727 deletions(-)
 create mode 100644 mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
 create mode 100644 mlir/test/Dialect/XeGPU/propagate-layout.mlir
 create mode 100644 mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
 delete mode 100644 mlir/test/Dialect/XeGPU/subgroup-distribution.mlir
 delete mode 100644 mlir/test/Dialect/XeGPU/subgroup-map-propagation.mlir

diff --git a/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td b/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
index 8bdf19ac0e47..3a88dae041dd 100644
--- a/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/XeGPU/Transforms/Passes.td
@@ -27,10 +27,23 @@ def XeGPUSubgroupDistribute : Pass<"xegpu-subgroup-distribute"> {
   }];
   let dependentDialects = ["memref::MemRefDialect", "xegpu::XeGPUDialect",
                            "vector::VectorDialect"];
+}
+
+def XeGPUPropagateLayout : Pass<"xegpu-propagate-layout"> {
+  let summary = "Propagate and assign XeGPU layout information";
+  let description = [{
+    This pass propagates the XeGPU layout information accross ops. Starting
+    from a set of anchor operations (e.g. `dpas`, `store_nd`), this will
+    propagate the layouts required for their operands to the producers. With
+    this propagated layout information, pass will then update op result type
+    with the layout information.
+  }];
+  let dependentDialects = ["memref::MemRefDialect", "xegpu::XeGPUDialect",
+                           "vector::VectorDialect"];
   let options = [Option<
-      "printOnly", "print-analysis-only", "bool",
-      /*default=*/"false",
-      "Print the result of the subgroup map propagation analysis and exit.">];
+    "printOnly", "print-analysis-only", "bool",
+    /*default=*/"false",
+    "Print the result of layout propagation analysis and exit.">];
 }
 
 def XeGPUWgToSgDistribute : Pass<"xegpu-wg-to-sg-distribute"> {
diff --git a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
index 6fea10185402..772cf7364964 100644
--- a/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
+++ b/mlir/include/mlir/Dialect/XeGPU/Utils/XeGPUUtils.h
@@ -24,6 +24,20 @@ class LayoutAttr;
 class TensorDescType;
 } // namespace xegpu
 
+namespace xegpu {
+/// HW dependent constants.
+/// TODO: These constants should be queried from the target information.
+namespace targetinfo {
+constexpr unsigned subgroupSize = 16; // How many lanes in a subgroup.
+/// If DPAS A or B operands have low precision element types they must be packed
+/// according to the following sizes.
+constexpr unsigned packedSizeInBitsForDefault =
+    16; // Minimum packing size per register for DPAS A.
+constexpr unsigned packedSizeInBitsForDpasB =
+    32; // Minimum packing size per register for DPAS B.
+} // namespace targetinfo
+} // namespace xegpu
+
 namespace xegpu {
 
 /// Flatten a set of ValueRange into a single SmallVector<Value>
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt b/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
index af0d7f6bd907..9c178d1d8564 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/XeGPU/Transforms/CMakeLists.txt
@@ -4,6 +4,7 @@ add_mlir_dialect_library(MLIRXeGPUTransforms
   XeGPUSubgroupDistribute.cpp
   XeGPUUnroll.cpp
   XeGPUWgToSgDistribute.cpp
+  XeGPUPropagateLayout.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/XeGPU
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
new file mode 100644
index 000000000000..cc22d2bbd8c3
--- /dev/null
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUPropagateLayout.cpp
@@ -0,0 +1,889 @@
+//===- XeGPUPropagateLayout.cpp - XeGPU Layout Propagation ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h"
+#include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h"
+#include "mlir/Analysis/DataFlow/SparseAnalysis.h"
+#include "mlir/Analysis/DataFlow/Utils.h"
+#include "mlir/Analysis/DataFlowFramework.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/MemRef/IR/MemRef.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Dialect/XeGPU/IR/XeGPU.h"
+#include "mlir/Dialect/XeGPU/Transforms/Passes.h"
+#include "mlir/Dialect/XeGPU/Utils/XeGPUUtils.h"
+#include "mlir/IR/Attributes.h"
+#include "mlir/IR/Builders.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/IR/BuiltinTypes.h"
+#include "mlir/IR/Operation.h"
+#include "mlir/IR/Value.h"
+#include "mlir/IR/Visitors.h"
+#include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Interfaces/FunctionInterfaces.h"
+#include "mlir/Support/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/TypeSwitch.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/InterleavedRange.h"
+#include "llvm/Support/LogicalResult.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace mlir {
+namespace xegpu {
+#define GEN_PASS_DEF_XEGPUPROPAGATELAYOUT
+#include "mlir/Dialect/XeGPU/Transforms/Passes.h.inc"
+} // namespace xegpu
+} // namespace mlir
+
+#define DEBUG_TYPE "xegpu-propagate-layout"
+#define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
+
+using namespace mlir;
+using namespace mlir::dataflow;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Layout
+//===----------------------------------------------------------------------===//
+
+/// Helper class to store the ND layout of lanes within a subgroup and data
+/// owned by each lane.
+struct Layout {
+  SmallVector<int64_t, 3> layout;
+  Layout() = default;
+  Layout(std::initializer_list<int64_t> list) : layout(list) {}
+  void print(llvm::raw_ostream &os) const;
+  size_t size() const { return layout.size(); }
+};
+
+void Layout::print(llvm::raw_ostream &os) const {
+  os << llvm::interleaved_array(layout);
+}
+
+/// LaneLayout represents the logical layout of lanes within a subgroup when it
+/// accesses some value. LaneData represents the logical layout of data owned by
+/// each work item.
+using LaneLayout = Layout;
+using LaneData = Layout;
+
+//===----------------------------------------------------------------------===//
+// LayoutInfo
+//===----------------------------------------------------------------------===//
+
+/// Helper class for tracking the analysis state of an mlir value. For layout
+/// propagation, the analysis state is simply the lane_layout and lane_data of
+/// each value. Purpose of this analysis to propagate some unique layout for
+/// each value in the program starting from a set of anchor operations (like
+/// DPAS, StoreNd, etc.).
+///
+/// Given this, LayoutInfo  satisifies the following properties:
+///  1) A LayoutInfo value can be in one of two states - `assigned` or `not
+///  assigned`.
+///  2) Two LayoutInfo values are equal if they are both assigned or
+///  both not assigned. The concrete value of assigned state does not matter.
+///  3) The meet operator works as follows:
+///     - If current state is assigned, return the current state. (already
+///     a unique layout is assigned. don't change it)
+///     - Otherwise, return the other state.
+
+struct LayoutInfo {
+private:
+  LaneLayout laneLayout;
+  LaneData laneData;
+  xegpu::LayoutAttr layoutAttr;
+
+public:
+  LayoutInfo() = default;
+  LayoutInfo(const LaneLayout &layout, const LaneData &data)
+      : laneLayout(layout), laneData(data) {}
+
+  // Two lattice values are equal if they have `some` layout. The actual
+  // content of the layout does not matter.
+  bool operator==(const LayoutInfo &other) const {
+    return this->isAssigned() == other.isAssigned();
+  }
+
+  static LayoutInfo meet(const LayoutInfo &lhs, const LayoutInfo &rhs);
+
+  static LayoutInfo join(const LayoutInfo &lhs, const LayoutInfo &rhs);
+
+  void print(raw_ostream &os) const;
+
+  bool isAssigned() const {
+    return laneLayout.size() > 0 && laneData.size() > 0;
+  }
+
+  LayoutInfo getTransposedLayout(ArrayRef<int64_t> permutation) const;
+
+  const LaneLayout &getLayout() const { return laneLayout; }
+  const LaneData &getData() const { return laneData; }
+  ArrayRef<int64_t> getLayoutAsArrayRef() const { return laneLayout.layout; }
+  ArrayRef<int64_t> getDataAsArrayRef() const { return laneData.layout; }
+};
+
+void LayoutInfo::print(raw_ostream &os) const {
+  if (isAssigned()) {
+    os << "lane_layout: ";
+    laneLayout.print(os);
+    os << ", lane_data: ";
+    laneData.print(os);
+  } else {
+    os << "Not assigned.";
+  }
+}
+
+LayoutInfo LayoutInfo::meet(const LayoutInfo &lhs, const LayoutInfo &rhs) {
+  if (!lhs.isAssigned())
+    return rhs;
+  return lhs;
+}
+
+/// Since this is a backward analysis, join method is not used.
+LayoutInfo LayoutInfo::join(const LayoutInfo &lhs, const LayoutInfo &rhs) {
+  llvm_unreachable("Join should not be triggered by layout propagation.");
+}
+
+/// Get the transposed layout according to the given permutation.
+LayoutInfo
+LayoutInfo::getTransposedLayout(ArrayRef<int64_t> permutation) const {
+  if (!isAssigned())
+    return {};
+  LaneLayout newLayout;
+  LaneData newData;
+  for (int64_t idx : permutation) {
+    newLayout.layout.push_back(laneLayout.layout[idx]);
+    newData.layout.push_back(laneData.layout[idx]);
+  }
+  return LayoutInfo(newLayout, newData);
+}
+
+//===----------------------------------------------------------------------===//
+// LayoutInfoLattice
+//===----------------------------------------------------------------------===//
+
+/// Lattice holding the LayoutInfo for each value.
+struct LayoutInfoLattice : public Lattice<LayoutInfo> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LayoutInfoLattice)
+  using Lattice::Lattice;
+};
+
+/// Helper Functions to get default layouts. A `default layout` is a layout that
+/// is assigned to a value when the layout is not fixed by some anchor operation
+/// (like DPAS).
+
+/// Helper Function to get the default layout for uniform values like constants.
+/// For 1D vector, lane_layout is [subgroupSize] and lane_data is [1].
+/// For 2D vector, lane_layout is [1, subgroupSize] and lane_data is [1, 1].
+static LayoutInfo getDefaultSIMTLayoutInfo(unsigned rank) {
+  assert((rank == 1 || rank == 2) && "Expected 1D or 2D vector.");
+  if (rank == 1)
+    return LayoutInfo(LaneLayout({xegpu::targetinfo::subgroupSize}),
+                      LaneData({1}));
+  return LayoutInfo(LaneLayout({1, xegpu::targetinfo::subgroupSize}),
+                    LaneData({1, 1}));
+}
+
+/// Helper to get the default layout for a vector type.
+static LayoutInfo getDefaultSIMTLayoutInfo(VectorType vectorTy) {
+  // Expecting a 1D or 2D vector.
+  assert((vectorTy.getRank() == 1 || vectorTy.getRank() == 2) &&
+         "Expected 1D or 2D vector.");
+  // Expecting int or float element type.
+  assert(vectorTy.getElementType().isIntOrFloat() &&
+         "Expected int or float element type.");
+  // If the rank is 1, then return default layout for 1D vector.
+  if (vectorTy.getRank() == 1)
+    return getDefaultSIMTLayoutInfo(1);
+  // Packing factor is determined by the element type bitwidth.
+  int packingFactor = 1;
+  unsigned bitwidth = vectorTy.getElementType().getIntOrFloatBitWidth();
+  if (bitwidth < xegpu::targetinfo::packedSizeInBitsForDefault)
+    packingFactor = xegpu::targetinfo::packedSizeInBitsForDefault / bitwidth;
+  return LayoutInfo(LaneLayout({1, xegpu::targetinfo::subgroupSize}),
+                    LaneData({1, packingFactor}));
+}
+
+/// Helper Function to get the expected layouts for DPAS operands. `lane_data`
+/// is set according to the following criteria:
+/// * For A operand, the data must be packed in minimum
+/// `packedSizeInBitsForDefault`
+/// * For B operand, the data must be packed in minimum
+/// `packedSizeInBitsForDpasB`
+static LayoutInfo getSIMTLayoutInfoForDPASOperand(VectorType vectorTy,
+                                                  unsigned operandNum) {
+  Type elementTy = vectorTy.getElementType();
+  assert(elementTy.isIntOrFloat() &&
+         "Expected int or float type in DPAS operands");
+  LaneLayout layout({1, xegpu::targetinfo::subgroupSize});
+  // For B operand, data must be packed in minimum `packedDpasBSizeInBits` and
+  // must have the VNNI format.
+  if (operandNum == 1 && elementTy.getIntOrFloatBitWidth() <
+                             xegpu::targetinfo::packedSizeInBitsForDpasB) {
+    LaneData data({xegpu::targetinfo::packedSizeInBitsForDpasB /
+                       elementTy.getIntOrFloatBitWidth(),
+                   1});
+    return LayoutInfo(layout, data);
+  }
+  // Otherwise, return the default layout for the vector type.
+  return getDefaultSIMTLayoutInfo(vectorTy);
+}
+
+//===----------------------------------------------------------------------===//
+// LayoutInfoPropagation
+//===----------------------------------------------------------------------===//
+
+/// Backward data flow analysis to propagate the lane_layout and lane_data of
+/// each value in the program. Currently, the layouts for operands DPAS,
+/// StoreNd, and StoreScatter are fixed (known before propagation). Purpose of
+/// this analysis is to propagate those known layouts to all their producers and
+/// (other) consumers.
+class LayoutInfoPropagation
+    : public SparseBackwardDataFlowAnalysis<LayoutInfoLattice> {
+private:
+  void visitDpasOp(xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
+                   ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitStoreNdOp(xegpu::StoreNdOp store,
+                      ArrayRef<LayoutInfoLattice *> operands,
+                      ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitStoreScatterOp(xegpu::StoreScatterOp storeScatter,
+                           ArrayRef<LayoutInfoLattice *> operands,
+                           ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitLoadNdOp(xegpu::LoadNdOp load,
+                     ArrayRef<LayoutInfoLattice *> operands,
+                     ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitLoadGatherOp(xegpu::LoadGatherOp load,
+                         ArrayRef<LayoutInfoLattice *> operands,
+                         ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitTransposeOp(vector::TransposeOp transpose,
+                        ArrayRef<LayoutInfoLattice *> operands,
+                        ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitVectorBitcastOp(vector::BitCastOp bitcast,
+                            ArrayRef<LayoutInfoLattice *> operands,
+                            ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitCreateDescOp(xegpu::CreateDescOp createDesc,
+                         ArrayRef<LayoutInfoLattice *> operands,
+                         ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitUpdateNdOffsetOp(xegpu::UpdateNdOffsetOp updateNdOffset,
+                             ArrayRef<LayoutInfoLattice *> operands,
+                             ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitPrefetchNdOp(xegpu::PrefetchNdOp prefetch,
+                         ArrayRef<LayoutInfoLattice *> operands,
+                         ArrayRef<const LayoutInfoLattice *> results);
+
+  void visitVectorMultiReductionOp(vector::MultiDimReductionOp reduction,
+                                   ArrayRef<LayoutInfoLattice *> operands,
+                                   ArrayRef<const LayoutInfoLattice *> results);
+
+public:
+  LayoutInfoPropagation(DataFlowSolver &solver,
+                        SymbolTableCollection &symbolTable)
+      : SparseBackwardDataFlowAnalysis(solver, symbolTable) {}
+  using SparseBackwardDataFlowAnalysis::SparseBackwardDataFlowAnalysis;
+
+  LogicalResult
+  visitOperation(Operation *op, ArrayRef<LayoutInfoLattice *> operands,
+                 ArrayRef<const LayoutInfoLattice *> results) override;
+
+  void visitBranchOperand(OpOperand &operand) override {};
+
+  void visitCallOperand(OpOperand &operand) override {};
+
+  void visitExternalCall(CallOpInterface call,
+                         ArrayRef<LayoutInfoLattice *> operands,
+                         ArrayRef<const LayoutInfoLattice *> results) override {
+  };
+
+  void setToExitState(LayoutInfoLattice *lattice) override {
+    (void)lattice->meet(LayoutInfo());
+  }
+};
+} // namespace
+
+LogicalResult LayoutInfoPropagation::visitOperation(
+    Operation *op, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  TypeSwitch<Operation *>(op)
+      .Case<xegpu::DpasOp>(
+          [&](auto dpasOp) { visitDpasOp(dpasOp, operands, results); })
+      .Case<xegpu::StoreNdOp>(
+          [&](auto storeNdOp) { visitStoreNdOp(storeNdOp, operands, results); })
+      .Case<xegpu::StoreScatterOp>([&](auto storeScatterOp) {
+        visitStoreScatterOp(storeScatterOp, operands, results);
+      })
+      .Case<xegpu::LoadNdOp>(
+          [&](auto loadNdOp) { visitLoadNdOp(loadNdOp, operands, results); })
+      .Case<xegpu::LoadGatherOp>([&](auto loadGatherOp) {
+        visitLoadGatherOp(loadGatherOp, operands, results);
+      })
+      .Case<xegpu::CreateDescOp>([&](auto createDescOp) {
+        visitCreateDescOp(createDescOp, operands, results);
+      })
+      .Case<xegpu::UpdateNdOffsetOp>([&](auto updateNdOffsetOp) {
+        visitUpdateNdOffsetOp(updateNdOffsetOp, operands, results);
+      })
+      .Case<xegpu::PrefetchNdOp>([&](auto prefetchNdOp) {
+        visitPrefetchNdOp(prefetchNdOp, operands, results);
+      })
+      .Case<vector::TransposeOp>([&](auto transposeOp) {
+        visitTransposeOp(transposeOp, operands, results);
+      })
+      .Case<vector::BitCastOp>([&](auto bitcastOp) {
+        visitVectorBitcastOp(bitcastOp, operands, results);
+      })
+      .Case<vector::MultiDimReductionOp>([&](auto reductionOp) {
+        visitVectorMultiReductionOp(reductionOp, operands, results);
+      })
+      // All other ops.
+      .Default([&](Operation *op) {
+        for (const LayoutInfoLattice *resultInfo : results) {
+          if (!resultInfo->getValue().isAssigned())
+            continue;
+          for (auto [operandInfo, operand] :
+               llvm::zip(operands, op->getOpOperands())) {
+            // If the operand type is not a vector or tensor descriptor, skip
+            // it.
+            if (!isa<xegpu::TensorDescType, VectorType>(
+                    operand.get().getType()))
+              continue;
+            // Propagate the result layout to the operand.
+            meet(operandInfo, *resultInfo);
+          }
+        }
+      });
+
+  return success();
+}
+
+void LayoutInfoPropagation::visitPrefetchNdOp(
+    xegpu::PrefetchNdOp prefetch, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // Here we assign the default layout to the tensor descriptor operand of
+  // prefetch.
+  auto tdescTy = prefetch.getTensorDescType();
+  auto prefetchLayout = getDefaultSIMTLayoutInfo(
+      VectorType::get(tdescTy.getShape(), tdescTy.getElementType()));
+  // Propagate the layout to the source tensor descriptor.
+  propagateIfChanged(operands[0], operands[0]->meet(prefetchLayout));
+}
+
+void LayoutInfoPropagation::visitVectorMultiReductionOp(
+    vector::MultiDimReductionOp reduction,
+    ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // The layout of the result must be present.
+  LayoutInfo resultLayout = results[0]->getValue();
+  if (!resultLayout.isAssigned())
+    return;
+  // We only consider 2D -> 1D reductions at this point.
+  VectorType resultTy = llvm::dyn_cast<VectorType>(reduction.getDestType());
+  if (!resultTy || resultTy.getRank() != 1) {
+    reduction.emitWarning("Expecting output type to be 1D vector.");
+    return;
+  }
+  // Given that the result is 1D, the layout of the operand should be 2D with
+  // default layout.
+  LayoutInfo operandLayout = getDefaultSIMTLayoutInfo(2);
+  propagateIfChanged(operands[0], operands[0]->meet(operandLayout));
+  // Accumulator should have the same layout as the result.
+  propagateIfChanged(operands[1], operands[1]->meet(resultLayout));
+}
+
+/// Propagate the layout of the result tensor to the source tensor descriptor in
+/// UpdateNdOffsetOp.
+void LayoutInfoPropagation::visitUpdateNdOffsetOp(
+    xegpu::UpdateNdOffsetOp updateNdOffset,
+    ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // The layout of the result must be present.
+  LayoutInfo resultLayout = results[0]->getValue();
+  if (!resultLayout.isAssigned())
+    return;
+  // Propagate the layout to the source operand.
+  propagateIfChanged(operands[0], operands[0]->meet(resultLayout));
+}
+
+/// Set the layouts for DPAS A, B, and C operands.
+void LayoutInfoPropagation::visitDpasOp(
+    xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  VectorType aTy = dpas.getLhsType();
+  VectorType bTy = dpas.getRhsType();
+  propagateIfChanged(
+      operands[0], operands[0]->meet(getSIMTLayoutInfoForDPASOperand(aTy, 0)));
+  propagateIfChanged(
+      operands[1], operands[1]->meet(getSIMTLayoutInfoForDPASOperand(bTy, 1)));
+  if (operands.size() > 2) {
+    VectorType cTy = dpas.getAccType();
+    propagateIfChanged(
+        operands[2],
+        operands[2]->meet(getSIMTLayoutInfoForDPASOperand(cTy, 2)));
+  }
+}
+
+/// Set the layout for the value and tensor descriptor operands in StoreNdOp.
+void LayoutInfoPropagation::visitStoreNdOp(
+    xegpu::StoreNdOp store, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  LayoutInfo storeLayout = getDefaultSIMTLayoutInfo(store.getValueType());
+  // Both operands should have the same layout
+  for (LayoutInfoLattice *operand : operands)
+    propagateIfChanged(operand, operand->meet(storeLayout));
+}
+
+/// Propagate the layout of the value to the tensor descriptor operand in
+/// LoadNdOp.
+void LayoutInfoPropagation::visitLoadNdOp(
+    xegpu::LoadNdOp load, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  LayoutInfo valueLayout = results[0]->getValue();
+  // Need the layout of the value to propagate to the tensor descriptor.
+  if (!valueLayout.isAssigned())
+    return;
+  LayoutInfo tensorDescLayout = valueLayout;
+  // LoadNdOp has the transpose effect. However, at the stage of this analysis
+  // this effect is not expected and should be abstracted away. Emit a
+  // warning.
+  if (auto transpose = load.getTranspose()) {
+    load.emitWarning("Transpose effect is not expected for LoadNdOp at "
+                     "LayoutInfoPropagation stage.");
+    tensorDescLayout = valueLayout.getTransposedLayout(transpose.value());
+  }
+  // Propagate the new layout to the tensor descriptor operand.
+  propagateIfChanged(operands[0], operands[0]->meet(tensorDescLayout));
+}
+
+/// For vector::TransposeOp, the layout of the result is transposed and
+/// propagated to the operand.
+void LayoutInfoPropagation::visitTransposeOp(
+    vector::TransposeOp transpose, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // Need the layout of transpose result to propagate to the operands.
+  LayoutInfo resultLayout = results[0]->getValue();
+  if (!resultLayout.isAssigned())
+    return;
+  LayoutInfo newLayout =
+      resultLayout.getTransposedLayout(transpose.getPermutation());
+  // Propagate the new layout to the vector operand.
+  propagateIfChanged(operands[0], operands[0]->meet(newLayout));
+}
+
+/// For vector::BitCastOp, the lane_data of the source layout is changed based
+/// on the bit width of the source and result types.
+void LayoutInfoPropagation::visitVectorBitcastOp(
+    vector::BitCastOp bitcast, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // Need the layout of bitcast result to propagate to the operands.
+  LayoutInfo resultLayout = results[0]->getValue();
+  if (!resultLayout.isAssigned())
+    return;
+  int inElemTyBitWidth =
+      bitcast.getSourceVectorType().getElementType().getIntOrFloatBitWidth();
+  int outElemTyBitWidth =
+      bitcast.getResultVectorType().getElementType().getIntOrFloatBitWidth();
+
+  // NOTE: We do not expect widening or narrowing bitcasts at this stage. Emit
+  // a warning and return.
+  if (inElemTyBitWidth != outElemTyBitWidth) {
+    bitcast.emitWarning("Widening or narrowing bitcasts are not expected at "
+                        "layout propagation stage.");
+    return;
+  }
+
+  propagateIfChanged(operands[0], operands[0]->meet(resultLayout));
+}
+
+/// Propagate the layout of the result to the tensor descriptor and mask
+/// operands in LoadGatherOp.
+void LayoutInfoPropagation::visitLoadGatherOp(
+    xegpu::LoadGatherOp load, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  LayoutInfo valueLayout = results[0]->getValue();
+  // Need the layout of the value to propagate to the tensor descriptor.
+  if (!valueLayout.isAssigned())
+    return;
+
+  LayoutInfo tensorDescLayout = valueLayout;
+  if (load.getTranspose()) {
+    // LoadGatherOp has the transpose effect. However, at the stage of this
+    // analyis this effect is not expected and should be abstracted away. Emit
+    // a warning.
+    load.emitWarning("Transpose effect is not expected for LoadGatherOp at "
+                     "LayoutInfoPropagation stage.");
+    tensorDescLayout = valueLayout.getTransposedLayout({1, 0});
+  }
+  // Mask operand should have 1D default layout.
+  LayoutInfo maskLayout = getDefaultSIMTLayoutInfo(1);
+  // Propagate the new layout to the tensor descriptor operand.
+  propagateIfChanged(operands[0], operands[0]->meet(tensorDescLayout));
+  // Propagate the new layout to the mask operand.
+  propagateIfChanged(operands[1], operands[1]->meet(maskLayout));
+}
+
+/// Propagate the layout of the descriptor to the vector offset operand in
+/// CreateDescOp.
+void LayoutInfoPropagation::visitCreateDescOp(
+    xegpu::CreateDescOp createDesc, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  LayoutInfo descLayout = results[0]->getValue();
+  // Need the layout of the descriptor to propagate to the operands.
+  if (!descLayout.isAssigned())
+    return;
+  // For offset operand propagate 1D default layout.
+  LayoutInfo layout = getDefaultSIMTLayoutInfo(1);
+  propagateIfChanged(operands[1], operands[1]->meet(layout));
+}
+
+/// Set the layout for the value, tensor descriptor, and mask operands in the
+/// StoreScatterOp.
+void LayoutInfoPropagation::visitStoreScatterOp(
+    xegpu::StoreScatterOp storeScatter, ArrayRef<LayoutInfoLattice *> operands,
+    ArrayRef<const LayoutInfoLattice *> results) {
+  // Currently, for 2D StoreScatterOp we expect that the height dimension of
+  // the tensor descriptor is equal to the subgroup size. This is ensured by
+  // the op verifier.
+  ArrayRef<int64_t> tdescShape = storeScatter.getTensorDescType().getShape();
+  if (tdescShape.size() > 1)
+    assert(
+        tdescShape[0] == xegpu::targetinfo::subgroupSize &&
+        "Expected the first dimension of 2D tensor descriptor to be equal to "
+        "subgroup size.");
+
+  LayoutInfo valueLayout =
+      getDefaultSIMTLayoutInfo(storeScatter.getValueType());
+  LayoutInfo storeScatterLayout = valueLayout;
+  if (storeScatter.getTranspose()) {
+    // StoreScatteOp allows transpose effect. However, at the stage of this
+    // analyis this effect is not expected and should be abstracted away. Emit
+    // a warning.
+    storeScatter.emitWarning("Transpose effect is not expected for "
+                             "StoreScatterOp at LayoutInfoPropagation stage.");
+    storeScatterLayout = valueLayout.getTransposedLayout({1, 0});
+  }
+  // Propagate the value layout.
+  propagateIfChanged(operands[0], operands[0]->meet(valueLayout));
+  // Propagate the tensor descriptor layout.
+  propagateIfChanged(operands[1], operands[1]->meet(storeScatterLayout));
+  // Use default 1D layout for mask operand.
+  LayoutInfo maskLayout = getDefaultSIMTLayoutInfo(1);
+  propagateIfChanged(operands[2], operands[2]->meet(maskLayout));
+}
+
+namespace {
+//===----------------------------------------------------------------------===//
+// RunLayoutInfoPropagation
+//===----------------------------------------------------------------------===//
+
+/// Driver class for running the LayoutInfoPropagation analysis.
+class RunLayoutInfoPropagation {
+public:
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RunLayoutInfoPropagation)
+
+  RunLayoutInfoPropagation(Operation *op) : target(op) {
+    SymbolTableCollection symbolTable;
+    loadBaselineAnalyses(solver);
+    solver.load<LayoutInfoPropagation>(symbolTable);
+    (void)solver.initializeAndRun(op);
+  }
+
+  LayoutInfo getLayoutInfo(Value val);
+
+  void printAnalysisResult(llvm::raw_ostream &os);
+
+private:
+  DataFlowSolver solver;
+  const Operation *target;
+};
+} // namespace
+
+LayoutInfo RunLayoutInfoPropagation::getLayoutInfo(Value val) {
+  auto *state = solver.lookupState<LayoutInfoLattice>(val);
+  if (!state)
+    return {};
+  return state->getValue();
+}
+
+// Print the analysis result for debugging purposes.
+void RunLayoutInfoPropagation::printAnalysisResult(llvm::raw_ostream &os) {
+  auto printFunctionResult = [&](FunctionOpInterface funcOp) {
+    os << "function: " << funcOp.getName() << ":\n";
+    // Function arguments
+    for (BlockArgument arg : funcOp.getArguments()) {
+      LayoutInfo layout = getLayoutInfo(arg);
+      os << "argument: " << arg << "\n";
+      os << "layout  : ";
+      layout.print(os);
+      os << "\n";
+    }
+    // Function ops
+    funcOp.walk([&](Operation *op) {
+      // Skip ops that do not have results
+      if (op->getResults().empty())
+        return;
+      os << "op    : ";
+      // For control-flow ops, print the op name only.
+      if (isa<BranchOpInterface>(op) || isa<RegionBranchOpInterface>(op))
+        os << op->getName();
+      else
+        op->print(os);
+      os << "\n";
+      // Print the layout for each result.
+      for (auto [i, r] : llvm::enumerate(op->getResults())) {
+        LayoutInfo layout = getLayoutInfo(r);
+        os << "layout for result #" << i << ": ";
+        layout.print(os);
+        os << "\n";
+      }
+    });
+  };
+
+  SmallVector<FunctionOpInterface> funcOps;
+  if (auto modOp = dyn_cast<ModuleOp>(target)) {
+    for (auto funcOp : modOp.getOps<FunctionOpInterface>())
+      funcOps.push_back(funcOp);
+
+    // Collect all GpuFuncOps in the module.
+    for (auto gpuModOp : modOp.getOps<gpu::GPUModuleOp>()) {
+      for (auto gpuFuncOp : gpuModOp.getOps<FunctionOpInterface>())
+        funcOps.push_back(gpuFuncOp);
+    }
+  }
+  // Print the analysis result for each function.
+  for (FunctionOpInterface funcOp : funcOps)
+    printFunctionResult(funcOp);
+}
+
+using GetLayoutFnTy = function_ref<xegpu::LayoutAttr(Value)>;
+/// Update an operation with the layout of its results. If the result type is a
+/// vector type, a temporary layout attribute is added to the operation. If the
+/// result type is a tensor descriptor type, the type is updated with the layout
+/// attribute. The users of the result are also updated with the layout
+/// attribute.
+static LogicalResult updateOp(mlir::OpBuilder &builder, mlir::Operation *op,
+                              GetLayoutFnTy getLayoutOfValue) {
+  // Region ops (like scf.for) are already handled by the updateControlFlowOps.
+  if (mlir::isa<mlir::RegionBranchOpInterface>(op))
+    return success();
+
+  // Iterate over all the results.
+  for (OpResult result : op->getResults()) {
+    Type resultType = result.getType();
+    // Layouts are needed only for vector and tensor descriptor types.
+    if (!isa<VectorType, xegpu::TensorDescType>(resultType))
+      continue;
+    // If the result has no layout but has users, emit a warning and continue.
+    xegpu::LayoutAttr layout = getLayoutOfValue(result);
+    if (!layout && result.getNumUses() > 0) {
+      op->emitWarning("op has users but no layout assigned for its result");
+      continue;
+    }
+    // If the result is a tensor descriptor type, update the tensor desc type
+    // with layout.
+    if (auto tensorDescTy = dyn_cast<xegpu::TensorDescType>(resultType)) {
+      auto typeWithLayout = xegpu::TensorDescType::get(
+          tensorDescTy.getContext(), tensorDescTy.getShape(),
+          tensorDescTy.getElementType(), tensorDescTy.getEncoding(), layout);
+      result.setType(typeWithLayout);
+      continue;
+    }
+    // If the result is a vector type, add a temporary layout attribute to the
+    // op.
+    xegpu::setLayoutAttr(result, layout);
+  }
+  return success();
+}
+
+/// Region ops like scf.for need special handling because they have blocks
+/// inside. If the blocks have tensor descriptor type as block arguments, thier
+/// types must be updated. Also region op can have results that may not have any
+/// users (e.g. A and B tiles). They are not assigned a layout by layout
+/// analysis because they have no users. However inside the region op
+/// corresponding block arguments for these results do have layouts. Therefore,
+/// in this case we still need to update the result types with the layout
+/// attribute. This function function updates the internal block arguments and
+/// the result types of the region op with the assigned layouts.
+/// clang-format off
+/// Example: scf.for ... iter_args(...) -> (out types) {
+///   ^bb0(block types):
+///     ...
+///   scf.yield ... : (yield types)
+/// }
+/// clang-format on
+/// In this example, at scf.yield, control-flow can transfer to two successor
+/// regions. One is the ^bb0 (for loop body) and the other is the scf.for op
+/// itself (yield the results). So we update both the block arguments of the
+/// successor region (i.e. block types) and the result types of the scf.for op
+/// (i.e. out types). Note that yield types are updated by respective producers
+/// inside bb0.
+static LogicalResult
+updateControlFlowOps(mlir::OpBuilder &builder,
+                     mlir::RegionBranchTerminatorOpInterface terminator,
+                     GetLayoutFnTy getLayoutOfValue) {
+  // Only process if the terminator is inside a region branch op.
+  if (!mlir::isa<mlir::RegionBranchOpInterface>(terminator->getParentOp()))
+    return success();
+
+  llvm::SmallVector<mlir::RegionSuccessor> successors;
+  llvm::SmallVector<mlir::Attribute> operands(terminator->getNumOperands(),
+                                              nullptr);
+  terminator.getSuccessorRegions(operands, successors);
+
+  for (mlir::RegionSuccessor &successor : successors) {
+    mlir::OperandRange successorOperands =
+        terminator.getSuccessorOperands(successor);
+    mlir::ValueRange successorInputs = successor.getSuccessorInputs();
+    for (auto [successorOperand, successorInput] :
+         llvm::zip(successorOperands, successorInputs)) {
+      Type inputType = successorInput.getType();
+      // We only need to operate on tensor descriptor or vector types.
+      if (!isa<xegpu::TensorDescType, VectorType>(inputType))
+        continue;
+      xegpu::LayoutAttr successorInputLayout = getLayoutOfValue(successorInput);
+      xegpu::LayoutAttr successorOperandLayout =
+          getLayoutOfValue(successorOperand);
+
+      // If either of the layouts is not assigned, we cannot proceed.
+      if (!successorOperandLayout) {
+        LLVM_DEBUG(
+            DBGS()
+            << "No layout assigned for forwarded operand in branch terminator: "
+            << successorOperand << "\n");
+        return failure();
+      }
+      // We expect the layouts to match.
+      if (successorInputLayout &&
+          successorInputLayout != successorOperandLayout) {
+        LLVM_DEBUG(DBGS() << "Conflicting layouts for region argument and "
+                             "operand forwarded as the argument: "
+                          << successorInputLayout << " vs "
+                          << successorOperandLayout << "\n");
+        return failure();
+      }
+      // Get tensor descriptor type with the layout.
+      if (auto tdescTy = dyn_cast<xegpu::TensorDescType>(inputType)) {
+        auto newTdescTy = xegpu::TensorDescType::get(
+            tdescTy.getContext(), tdescTy.getShape(), tdescTy.getElementType(),
+            tdescTy.getEncoding(), successorOperandLayout);
+        successorInput.setType(newTdescTy);
+        continue;
+      }
+      // If the type is a vector type and this region argument is an OpResult,
+      // set the layout attribute on the OpResult.
+      if (auto result = dyn_cast<OpResult>(successorInput))
+        xegpu::setLayoutAttr(result, successorOperandLayout);
+    }
+  }
+  return success();
+}
+
+/// Update the function arguments and results with the layouts.
+static LogicalResult updateFunctionOpInterface(mlir::OpBuilder &builder,
+                                               mlir::FunctionOpInterface funcOp,
+                                               GetLayoutFnTy getLayoutOfValue) {
+  SmallVector<Type> newArgTypes;
+  // Update the function arguments.
+  for (BlockArgument arg : funcOp.getArguments()) {
+    Type argType = arg.getType();
+    newArgTypes.push_back(argType);
+    if (!isa<VectorType, xegpu::TensorDescType>(argType))
+      continue;
+    xegpu::LayoutAttr layout = getLayoutOfValue(arg);
+    if (!layout) {
+      LLVM_DEBUG(DBGS() << "Expecting layout for function argument: " << arg
+                        << " but got none.\n");
+      return failure();
+    }
+    if (auto tensorDescTy = dyn_cast<xegpu::TensorDescType>(argType)) {
+      auto newTdescTy = xegpu::TensorDescType::get(
+          tensorDescTy.getContext(), tensorDescTy.getShape(),
+          tensorDescTy.getElementType(), tensorDescTy.getEncoding(), layout);
+      arg.setType(newTdescTy);
+      newArgTypes.back() = newTdescTy;
+    }
+  }
+  // Update the function type with the new argument types.
+  // NOTE: We assume that function results are not expected to have layouts.
+  funcOp.setType(FunctionType::get(funcOp.getContext(), newArgTypes,
+                                   funcOp.getResultTypes()));
+  return success();
+}
+
+namespace {
+struct XeGPUPropagateLayoutPass final
+    : public xegpu::impl::XeGPUPropagateLayoutBase<XeGPUPropagateLayoutPass> {
+  XeGPUPropagateLayoutPass() = default;
+  XeGPUPropagateLayoutPass(const XeGPUPropagateLayoutPass &other) = default;
+  XeGPUPropagateLayoutPass(xegpu::XeGPUPropagateLayoutOptions options)
+      : XeGPUPropagateLayoutBase(options) {}
+  void runOnOperation() override;
+};
+
+} // namespace
+
+void XeGPUPropagateLayoutPass::runOnOperation() {
+  auto &analysis = getAnalysis<RunLayoutInfoPropagation>();
+  // Print the analysis result and exit. (for debugging purposes)
+  if (printOnly) {
+    auto &os = llvm::outs();
+    analysis.printAnalysisResult(os);
+    return;
+  }
+  // Helper to convert LayoutInfo to xegpu::LayoutAttr.
+  auto getXeGPULayoutForValue = [&](Value val) -> xegpu::LayoutAttr {
+    LayoutInfo layout = analysis.getLayoutInfo(val);
+    if (!layout.isAssigned())
+      return {};
+    return xegpu::LayoutAttr::get(
+        val.getContext(), llvm::to_vector_of<int>(layout.getLayoutAsArrayRef()),
+        llvm::to_vector_of<int>(layout.getDataAsArrayRef()));
+  };
+
+  mlir::OpBuilder builder(&getContext());
+  Operation *op = getOperation();
+  auto walkResult = op->walk([&](mlir::Block *block) -> WalkResult {
+    for (mlir::Operation &op : llvm::reverse(block->getOperations())) {
+      LogicalResult r = success();
+      TypeSwitch<Operation *>(&op)
+          .Case<mlir::RegionBranchTerminatorOpInterface>(
+              [&](mlir::RegionBranchTerminatorOpInterface branchTermOp) {
+                r = updateControlFlowOps(builder, branchTermOp,
+                                         getXeGPULayoutForValue);
+              })
+          .Case<mlir::FunctionOpInterface>(
+              [&](mlir::FunctionOpInterface funcOp) {
+                r = updateFunctionOpInterface(builder, funcOp,
+                                              getXeGPULayoutForValue);
+              })
+          .Default([&](Operation *op) {
+            r = updateOp(builder, op, getXeGPULayoutForValue);
+          });
+      if (failed(r)) {
+        op.emitError("Failed to update operation with the layout.");
+        return WalkResult::interrupt();
+      }
+    }
+    return WalkResult::advance();
+  });
+  if (walkResult.wasInterrupted()) {
+    signalPassFailure();
+    return;
+  }
+}
diff --git a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
index 66d21dbdaf06..dabcae0bfe4b 100644
--- a/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
+++ b/mlir/lib/Dialect/XeGPU/Transforms/XeGPUSubgroupDistribute.cpp
@@ -5,9 +5,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-#include "mlir/Analysis/DataFlow/SparseAnalysis.h"
-#include "mlir/Analysis/DataFlow/Utils.h"
-#include "mlir/Analysis/DataFlowFramework.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
 #include "mlir/Dialect/GPU/Utils/DistributionUtils.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
@@ -29,15 +26,13 @@
 #include "mlir/IR/Value.h"
 #include "mlir/IR/Visitors.h"
 #include "mlir/Interfaces/FunctionInterfaces.h"
+#include "mlir/Support/LLVM.h"
+#include "mlir/Transforms/DialectConversion.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "mlir/Transforms/InliningUtils.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/TypeSwitch.h"
-#include "llvm/Support/FormatVariadic.h"
-#include "llvm/Support/InterleavedRange.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace mlir {
 namespace xegpu {
@@ -50,788 +45,11 @@ namespace xegpu {
 #define DBGS() (llvm::dbgs() << "[" DEBUG_TYPE "]: ")
 
 using namespace mlir;
-using namespace mlir::dataflow;
 
-/// HW dependent constants.
-/// TODO: These constants should be queried from the target information.
-constexpr unsigned subgroupSize = 16; // How many lanes in a subgroup.
-/// If DPAS A or B operands have low precision element types they must be packed
-/// according to the following sizes.
-constexpr unsigned packedSizeInBitsForDefault =
-    16; // Minimum packing size per register for DPAS A.
-constexpr unsigned packedSizeInBitsForDpasB =
-    32; // Minimum packing size per register for DPAS B.
-
-namespace {
-
-//===----------------------------------------------------------------------===//
-// Layout
-//===----------------------------------------------------------------------===//
-
-/// Helper class to store the ND layout of lanes within a subgroup and data
-/// owned by each lane.
-struct Layout {
-  SmallVector<int64_t, 3> layout;
-  Layout() = default;
-  Layout(std::initializer_list<int64_t> list) : layout(list) {}
-  void print(llvm::raw_ostream &os) const;
-  size_t size() const { return layout.size(); }
-  int64_t operator[](size_t idx) const;
-};
-
-void Layout::print(llvm::raw_ostream &os) const {
-  os << llvm::interleaved_array(layout);
-}
-
-int64_t Layout::operator[](size_t idx) const {
-  assert(idx < layout.size() && "Index out of bounds.");
-  return layout[idx];
-}
-
-/// LaneLayout represents the logical layout of lanes within a subgroup when it
-/// accesses some value. LaneData represents the logical layout of data owned by
-/// each work item.
-using LaneLayout = Layout;
-using LaneData = Layout;
-
-//===----------------------------------------------------------------------===//
-// LayoutInfo
-//===----------------------------------------------------------------------===//
-
-/// Helper class for tracking the analysis state of an mlir value. For layout
-/// propagation, the analysis state is simply the lane_layout and lane_data of
-/// each value. Purpose of this analysis to propagate some unique layout for
-/// each value in the program starting from a set of anchor operations (like
-/// DPAS, StoreNd, etc.).
-///
-/// Given this, LayoutInfo  satisifies the following properties:
-///  1) A LayoutInfo value can be in one of two states - `assigned` or `not
-///  assigned`.
-///  2) Two LayoutInfo values are equal if they are both assigned or
-///  both not assigned. The concrete value of assigned state does not matter.
-///  3) The meet operator works as follows:
-///     - If current state is assigned, return the current state. (already
-///     a unique layout is assigned. don't change it)
-///     - Otherwise, return the other state.
-
-struct LayoutInfo {
-private:
-  LaneLayout laneLayout;
-  LaneData laneData;
-
-public:
-  LayoutInfo() = default;
-  LayoutInfo(const LaneLayout &layout, const LaneData &data)
-      : laneLayout(layout), laneData(data) {}
-
-  // Two lattice values are equal if they have `some` layout. The actual
-  // content of the layout does not matter.
-  bool operator==(const LayoutInfo &other) const {
-    return this->isAssigned() == other.isAssigned();
-  }
-
-  static LayoutInfo meet(const LayoutInfo &lhs, const LayoutInfo &rhs);
-
-  static LayoutInfo join(const LayoutInfo &lhs, const LayoutInfo &rhs);
-
-  void print(raw_ostream &os) const;
-
-  bool isAssigned() const {
-    return laneLayout.size() > 0 && laneData.size() > 0;
-  }
-
-  LayoutInfo getTransposedLayout(ArrayRef<int64_t> permutation) const;
-
-  const LaneLayout &getLayout() const { return laneLayout; }
-  const LaneData &getData() const { return laneData; }
-  ArrayRef<int64_t> getLayoutAsArrayRef() const { return laneLayout.layout; }
-  ArrayRef<int64_t> getDataAsArrayRef() const { return laneData.layout; }
-};
-
-void LayoutInfo::print(raw_ostream &os) const {
-  if (isAssigned()) {
-    os << "lane_layout: ";
-    laneLayout.print(os);
-    os << ", lane_data: ";
-    laneData.print(os);
-  } else {
-    os << "Not assigned.";
-  }
-}
-
-LayoutInfo LayoutInfo::meet(const LayoutInfo &lhs, const LayoutInfo &rhs) {
-  if (!lhs.isAssigned())
-    return rhs;
-  return lhs;
-}
-
-/// Since this is a backward analysis, join method is not used.
-LayoutInfo LayoutInfo::join(const LayoutInfo &lhs, const LayoutInfo &rhs) {
-  llvm_unreachable("Join should not be triggered by layout propagation.");
-}
-
-/// Get the transposed layout according to the given permutation.
-LayoutInfo
-LayoutInfo::getTransposedLayout(ArrayRef<int64_t> permutation) const {
-  if (!isAssigned())
-    return {};
-  LaneLayout newLayout;
-  LaneData newData;
-  for (int64_t idx : permutation) {
-    newLayout.layout.push_back(laneLayout.layout[idx]);
-    newData.layout.push_back(laneData.layout[idx]);
-  }
-  return LayoutInfo(newLayout, newData);
-}
-
-//===----------------------------------------------------------------------===//
-// LayoutInfoLattice
-//===----------------------------------------------------------------------===//
-
-/// Lattice holding the LayoutInfo for each value.
-struct LayoutInfoLattice : public Lattice<LayoutInfo> {
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(LayoutInfoLattice)
-  using Lattice::Lattice;
-};
-
-/// Helper Functions to get default layouts. A `default layout` is a layout that
-/// is assigned to a value when the layout is not fixed by some anchor operation
-/// (like DPAS).
-
-/// Helper Function to get the default layout for uniform values like constants.
-/// For 1D vector, lane_layout is [subgroupSize] and lane_data is [1].
-/// For 2D vector, lane_layout is [1, subgroupSize] and lane_data is [1, 1].
-static LayoutInfo getDefaultLayoutInfo(unsigned rank) {
-  assert((rank == 1 || rank == 2) && "Expected 1D or 2D vector.");
-  if (rank == 1)
-    return LayoutInfo(LaneLayout({subgroupSize}), LaneData({1}));
-  return LayoutInfo(LaneLayout({1, subgroupSize}), LaneData({1, 1}));
-}
-
-/// Helper to get the default layout for a vector type.
-static LayoutInfo getDefaultLayoutInfo(VectorType vectorTy) {
-  // Expecting a 1D or 2D vector.
-  assert((vectorTy.getRank() == 1 || vectorTy.getRank() == 2) &&
-         "Expected 1D or 2D vector.");
-  // Expecting int or float element type.
-  assert(vectorTy.getElementType().isIntOrFloat() &&
-         "Expected int or float element type.");
-  // If the rank is 1, then return default layout for 1D vector.
-  if (vectorTy.getRank() == 1)
-    return getDefaultLayoutInfo(1);
-  // Packing factor is determined by the element type bitwidth.
-  int packingFactor = 1;
-  unsigned bitwidth = vectorTy.getElementType().getIntOrFloatBitWidth();
-  if (bitwidth < packedSizeInBitsForDefault)
-    packingFactor = packedSizeInBitsForDefault / bitwidth;
-  return LayoutInfo(LaneLayout({1, subgroupSize}),
-                    LaneData({1, packingFactor}));
-}
-
-/// Helper Function to get the expected layouts for DPAS operands. `lane_data`
-/// is set according to the following criteria:
-/// * For A operand, the data must be packed in minimum
-/// `packedSizeInBitsForDefault`
-/// * For B operand, the data must be packed in minimum
-/// `packedSizeInBitsForDpasB`
-static LayoutInfo getLayoutInfoForDPASOperand(VectorType vectorTy,
-                                              unsigned operandNum) {
-  Type elementTy = vectorTy.getElementType();
-  assert(elementTy.isIntOrFloat() &&
-         "Expected int or float type in DPAS operands");
-  LaneLayout layout({1, subgroupSize});
-  // For B operand, data must be packed in minimum `packedDpasBSizeInBits` and
-  // must have the VNNI format.
-  if (operandNum == 1 &&
-      elementTy.getIntOrFloatBitWidth() < packedSizeInBitsForDpasB) {
-    LaneData data(
-        {packedSizeInBitsForDpasB / elementTy.getIntOrFloatBitWidth(), 1});
-    return LayoutInfo(layout, data);
-  }
-  // Otherwise, return the default layout for the vector type.
-  return getDefaultLayoutInfo(vectorTy);
-}
-
-//===----------------------------------------------------------------------===//
-// LayoutInfoPropagation
-//===----------------------------------------------------------------------===//
-
-/// Backward data flow analysis to propagate the lane_layout and lane_data of
-/// each value in the program. Currently, the layouts for operands DPAS,
-/// StoreNd, and StoreScatter are fixed (known before propagation). Purpose of
-/// this analysis is to propagate those known layouts to all their producers and
-/// (other) consumers.
-class LayoutInfoPropagation
-    : public SparseBackwardDataFlowAnalysis<LayoutInfoLattice> {
-private:
-  void visitDpasOp(xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
-                   ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitStoreNdOp(xegpu::StoreNdOp store,
-                      ArrayRef<LayoutInfoLattice *> operands,
-                      ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitStoreScatterOp(xegpu::StoreScatterOp storeScatter,
-                           ArrayRef<LayoutInfoLattice *> operands,
-                           ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitLoadNdOp(xegpu::LoadNdOp load,
-                     ArrayRef<LayoutInfoLattice *> operands,
-                     ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitLoadGatherOp(xegpu::LoadGatherOp load,
-                         ArrayRef<LayoutInfoLattice *> operands,
-                         ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitTransposeOp(vector::TransposeOp transpose,
-                        ArrayRef<LayoutInfoLattice *> operands,
-                        ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitVectorBitcastOp(vector::BitCastOp bitcast,
-                            ArrayRef<LayoutInfoLattice *> operands,
-                            ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitCreateDescOp(xegpu::CreateDescOp createDesc,
-                         ArrayRef<LayoutInfoLattice *> operands,
-                         ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitUpdateNdOffsetOp(xegpu::UpdateNdOffsetOp updateNdOffset,
-                             ArrayRef<LayoutInfoLattice *> operands,
-                             ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitPrefetchNdOp(xegpu::PrefetchNdOp prefetch,
-                         ArrayRef<LayoutInfoLattice *> operands,
-                         ArrayRef<const LayoutInfoLattice *> results);
-
-  void visitVectorMultiReductionOp(vector::MultiDimReductionOp reduction,
-                                   ArrayRef<LayoutInfoLattice *> operands,
-                                   ArrayRef<const LayoutInfoLattice *> results);
-
-public:
-  LayoutInfoPropagation(DataFlowSolver &solver,
-                        SymbolTableCollection &symbolTable)
-      : SparseBackwardDataFlowAnalysis(solver, symbolTable) {}
-  using SparseBackwardDataFlowAnalysis::SparseBackwardDataFlowAnalysis;
-
-  LogicalResult
-  visitOperation(Operation *op, ArrayRef<LayoutInfoLattice *> operands,
-                 ArrayRef<const LayoutInfoLattice *> results) override;
-
-  void visitBranchOperand(OpOperand &operand) override {};
-
-  void visitCallOperand(OpOperand &operand) override {};
-
-  void visitExternalCall(CallOpInterface call,
-                         ArrayRef<LayoutInfoLattice *> operands,
-                         ArrayRef<const LayoutInfoLattice *> results) override {
-  };
-
-  void setToExitState(LayoutInfoLattice *lattice) override {
-    (void)lattice->meet(LayoutInfo());
-  }
-};
-} // namespace
-
-LogicalResult LayoutInfoPropagation::visitOperation(
-    Operation *op, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  TypeSwitch<Operation *>(op)
-      .Case<xegpu::DpasOp>(
-          [&](auto dpasOp) { visitDpasOp(dpasOp, operands, results); })
-      .Case<xegpu::StoreNdOp>(
-          [&](auto storeNdOp) { visitStoreNdOp(storeNdOp, operands, results); })
-      .Case<xegpu::StoreScatterOp>([&](auto storeScatterOp) {
-        visitStoreScatterOp(storeScatterOp, operands, results);
-      })
-      .Case<xegpu::LoadNdOp>(
-          [&](auto loadNdOp) { visitLoadNdOp(loadNdOp, operands, results); })
-      .Case<xegpu::LoadGatherOp>([&](auto loadGatherOp) {
-        visitLoadGatherOp(loadGatherOp, operands, results);
-      })
-      .Case<xegpu::CreateDescOp>([&](auto createDescOp) {
-        visitCreateDescOp(createDescOp, operands, results);
-      })
-      .Case<xegpu::UpdateNdOffsetOp>([&](auto updateNdOffsetOp) {
-        visitUpdateNdOffsetOp(updateNdOffsetOp, operands, results);
-      })
-      .Case<xegpu::PrefetchNdOp>([&](auto prefetchNdOp) {
-        visitPrefetchNdOp(prefetchNdOp, operands, results);
-      })
-      // No need to propagate the layout to operands in CreateNdDescOp because
-      // they are scalars (offsets, sizes, etc.).
-      .Case<xegpu::CreateNdDescOp>([&](auto createNdDescOp) {})
-      .Case<vector::TransposeOp>([&](auto transposeOp) {
-        visitTransposeOp(transposeOp, operands, results);
-      })
-      .Case<vector::BitCastOp>([&](auto bitcastOp) {
-        visitVectorBitcastOp(bitcastOp, operands, results);
-      })
-      .Case<vector::MultiDimReductionOp>([&](auto reductionOp) {
-        visitVectorMultiReductionOp(reductionOp, operands, results);
-      })
-      // All other ops.
-      .Default([&](Operation *op) {
-        for (const LayoutInfoLattice *r : results) {
-          for (LayoutInfoLattice *operand : operands) {
-            // Propagate the layout of the result to the operand.
-            if (r->getValue().isAssigned())
-              meet(operand, *r);
-          }
-        }
-      });
-  // Add a dependency from each result to program point after the operation.
-  for (const LayoutInfoLattice *r : results) {
-    addDependency(const_cast<LayoutInfoLattice *>(r), getProgramPointAfter(op));
-  }
-  return success();
-}
-
-void LayoutInfoPropagation::visitPrefetchNdOp(
-    xegpu::PrefetchNdOp prefetch, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // Here we assign the default layout to the tensor descriptor operand of
-  // prefetch.
-  auto tdescTy = prefetch.getTensorDescType();
-  auto prefetchLayout = getDefaultLayoutInfo(
-      VectorType::get(tdescTy.getShape(), tdescTy.getElementType()));
-  // Propagate the layout to the source tensor descriptor.
-  propagateIfChanged(operands[0], operands[0]->meet(prefetchLayout));
-}
-
-void LayoutInfoPropagation::visitVectorMultiReductionOp(
-    vector::MultiDimReductionOp reduction,
-    ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // The layout of the result must be present.
-  LayoutInfo resultLayout = results[0]->getValue();
-  if (!resultLayout.isAssigned())
-    return;
-  // We only consider 2D -> 1D reductions at this point.
-  assert(resultLayout.getLayout().size() == 1 &&
-         "Expected 1D layout for reduction result.");
-  // Given that the result is 1D, the layout of the operand should be 2D with
-  // default layout.
-  LayoutInfo operandLayout = getDefaultLayoutInfo(2);
-  propagateIfChanged(operands[0], operands[0]->meet(operandLayout));
-  // Accumulator should have the same layout as the result.
-  propagateIfChanged(operands[1], operands[1]->meet(resultLayout));
-}
-
-/// Propagate the layout of the result tensor to the source tensor descriptor in
-/// UpdateNdOffsetOp.
-void LayoutInfoPropagation::visitUpdateNdOffsetOp(
-    xegpu::UpdateNdOffsetOp updateNdOffset,
-    ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // The layout of the result must be present.
-  LayoutInfo resultLayout = results[0]->getValue();
-  if (!resultLayout.isAssigned())
-    return;
-  // Propagate the layout to the source operand.
-  propagateIfChanged(operands[0], operands[0]->meet(resultLayout));
-}
-
-/// Set the layouts for DPAS A, B, and C operands.
-void LayoutInfoPropagation::visitDpasOp(
-    xegpu::DpasOp dpas, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  VectorType aTy = dpas.getLhsType();
-  VectorType bTy = dpas.getRhsType();
-  propagateIfChanged(operands[0],
-                     operands[0]->meet(getLayoutInfoForDPASOperand(aTy, 0)));
-  propagateIfChanged(operands[1],
-                     operands[1]->meet(getLayoutInfoForDPASOperand(bTy, 1)));
-  if (operands.size() > 2) {
-    VectorType cTy = dpas.getAccType();
-    propagateIfChanged(operands[2],
-                       operands[2]->meet(getLayoutInfoForDPASOperand(cTy, 2)));
-  }
-}
-
-/// Set the layout for the value and tensor descriptor operands in StoreNdOp.
-void LayoutInfoPropagation::visitStoreNdOp(
-    xegpu::StoreNdOp store, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  LayoutInfo storeLayout = getDefaultLayoutInfo(store.getValueType());
-  // Both operands should have the same layout
-  for (LayoutInfoLattice *operand : operands) {
-    propagateIfChanged(operand, operand->meet(storeLayout));
-  }
-}
-
-/// Propagate the layout of the value to the tensor descriptor operand in
-/// LoadNdOp.
-void LayoutInfoPropagation::visitLoadNdOp(
-    xegpu::LoadNdOp load, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  LayoutInfo valueLayout = results[0]->getValue();
-  // Need the layout of the value to propagate to the tensor descriptor.
-  if (!valueLayout.isAssigned())
-    return;
-  LayoutInfo tensorDescLayout = valueLayout;
-  // LoadNdOp has the transpose effect. However, at the stage of this analysis
-  // this effect is not expected and should be abstracted away. Emit a warning.
-  if (auto transpose = load.getTranspose()) {
-    load.emitWarning("Transpose effect is not expected for LoadNdOp at "
-                     "LayoutInfoPropagation stage.");
-    tensorDescLayout = valueLayout.getTransposedLayout(transpose.value());
-  }
-  // Propagate the new layout to the tensor descriptor operand.
-  propagateIfChanged(operands[0], operands[0]->meet(tensorDescLayout));
-}
-
-/// For vector::TransposeOp, the layout of the result is transposed and
-/// propagated to the operand.
-void LayoutInfoPropagation::visitTransposeOp(
-    vector::TransposeOp transpose, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // Need the layout of transpose result to propagate to the operands.
-  LayoutInfo resultLayout = results[0]->getValue();
-  if (!resultLayout.isAssigned())
-    return;
-  LayoutInfo newLayout =
-      resultLayout.getTransposedLayout(transpose.getPermutation());
-  // Propagate the new layout to the vector operand.
-  propagateIfChanged(operands[0], operands[0]->meet(newLayout));
-}
-
-/// For vector::BitCastOp, the lane_data of the source layout is changed based
-/// on the bit width of the source and result types.
-void LayoutInfoPropagation::visitVectorBitcastOp(
-    vector::BitCastOp bitcast, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // Need the layout of bitcast result to propagate to the operands.
-  LayoutInfo resultLayout = results[0]->getValue();
-  if (!resultLayout.isAssigned())
-    return;
-  int inElemTyBitWidth =
-      bitcast.getSourceVectorType().getElementType().getIntOrFloatBitWidth();
-  int outElemTyBitWidth =
-      bitcast.getResultVectorType().getElementType().getIntOrFloatBitWidth();
-
-  // LaneLayout does not change.
-  const LaneLayout &newLaneLayout = resultLayout.getLayout();
-  const LaneData &currData = resultLayout.getData();
-  LaneData newLaneData;
-  // It's a widening bitcast
-  if (inElemTyBitWidth < outElemTyBitWidth) {
-    int ratio = outElemTyBitWidth / inElemTyBitWidth;
-    newLaneData = resultLayout.getData()[0] == 1
-                      ? LaneData({1, currData[1] * ratio})
-                      : LaneData({currData[0] * ratio, 1});
-  } else {
-    // It's a narrowing bitcast
-    int ratio = inElemTyBitWidth / outElemTyBitWidth;
-    newLaneData = resultLayout.getData()[0] == 1
-                      ? LaneData({1, currData[1] / ratio})
-                      : LaneData({currData[0] / ratio, 1});
-  }
-
-  propagateIfChanged(operands[0],
-                     operands[0]->meet(LayoutInfo(newLaneLayout, newLaneData)));
-}
-
-/// Propagate the layout of the result to the tensor descriptor and mask
-/// operands in LoadGatherOp.
-void LayoutInfoPropagation::visitLoadGatherOp(
-    xegpu::LoadGatherOp load, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  LayoutInfo valueLayout = results[0]->getValue();
-  // Need the layout of the value to propagate to the tensor descriptor.
-  if (!valueLayout.isAssigned())
-    return;
-
-  LayoutInfo tensorDescLayout = valueLayout;
-  if (load.getTranspose()) {
-    // LoadGatherOp has the transpose effect. However, at the stage of this
-    // analyis this effect is not expected and should be abstracted away. Emit
-    // a warning.
-    load.emitWarning("Transpose effect is not expected for LoadGatherOp at "
-                     "LayoutInfoPropagation stage.");
-    tensorDescLayout = valueLayout.getTransposedLayout({1, 0});
-  }
-  // Mask operand should have 1D default layout.
-  LayoutInfo maskLayout = getDefaultLayoutInfo(1);
-  // Propagate the new layout to the tensor descriptor operand.
-  propagateIfChanged(operands[0], operands[0]->meet(tensorDescLayout));
-  // Propagate the new layout to the mask operand.
-  propagateIfChanged(operands[1], operands[1]->meet(maskLayout));
-}
-
-/// Propagate the layout of the descriptor to the vector offset operand in
-/// CreateDescOp.
-void LayoutInfoPropagation::visitCreateDescOp(
-    xegpu::CreateDescOp createDesc, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  LayoutInfo descLayout = results[0]->getValue();
-  // Need the layout of the descriptor to propagate to the operands.
-  if (!descLayout.isAssigned())
-    return;
-  // For offset operand propagate 1D default layout.
-  LayoutInfo layout = getDefaultLayoutInfo(1);
-  propagateIfChanged(operands[1], operands[1]->meet(layout));
-}
-
-/// Set the layout for the value, tensor descriptor, and mask operands in the
-/// StoreScatterOp.
-void LayoutInfoPropagation::visitStoreScatterOp(
-    xegpu::StoreScatterOp storeScatter, ArrayRef<LayoutInfoLattice *> operands,
-    ArrayRef<const LayoutInfoLattice *> results) {
-  // Currently, for 2D StoreScatterOp we expect that the height dimension of
-  // the tensor descriptor is equal to the subgroup size. This is ensured by
-  // the op verifier.
-  ArrayRef<int64_t> tdescShape = storeScatter.getTensorDescType().getShape();
-  if (tdescShape.size() > 1)
-    assert(
-        tdescShape[0] == subgroupSize &&
-        "Expected the first dimension of 2D tensor descriptor to be equal to "
-        "subgroup size.");
-
-  LayoutInfo valueLayout = getDefaultLayoutInfo(storeScatter.getValueType());
-  LayoutInfo storeScatterLayout = valueLayout;
-  if (storeScatter.getTranspose()) {
-    // StoreScatteOp allows transpose effect. However, at the stage of this
-    // analyis this effect is not expected and should be abstracted away. Emit
-    // a warning.
-    storeScatter.emitWarning("Transpose effect is not expected for "
-                             "StoreScatterOp at LayoutInfoPropagation stage.");
-    storeScatterLayout = valueLayout.getTransposedLayout({1, 0});
-  }
-  // Propagate the value layout.
-  propagateIfChanged(operands[0], operands[0]->meet(valueLayout));
-  // Propagate the tensor descriptor layout.
-  propagateIfChanged(operands[1], operands[1]->meet(storeScatterLayout));
-  // Use default 1D layout for mask operand.
-  LayoutInfo maskLayout = getDefaultLayoutInfo(1);
-  propagateIfChanged(operands[2], operands[2]->meet(maskLayout));
-}
-
-namespace {
-
-//===----------------------------------------------------------------------===//
-// RunLayoutInfoPropagation
-//===----------------------------------------------------------------------===//
-
-/// Driver class for running the LayoutInfoPropagation analysis.
-class RunLayoutInfoPropagation {
-public:
-  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(RunLayoutInfoPropagation)
-
-  RunLayoutInfoPropagation(Operation *op) : target(op) {
-    SymbolTableCollection symbolTable;
-    loadBaselineAnalyses(solver);
-    solver.load<LayoutInfoPropagation>(symbolTable);
-    (void)solver.initializeAndRun(op);
-  }
-
-  LayoutInfo getLayoutInfo(Value val);
-
-  void printAnalysisResult(llvm::raw_ostream &os);
-
-private:
-  DataFlowSolver solver;
-  const Operation *target;
-};
-} // namespace
-
-LayoutInfo RunLayoutInfoPropagation::getLayoutInfo(Value val) {
-  auto *state = solver.lookupState<LayoutInfoLattice>(val);
-  if (!state)
-    return {};
-  return state->getValue();
-}
-
-void RunLayoutInfoPropagation::printAnalysisResult(llvm::raw_ostream &os) {
-  auto printFunctionResult = [&](FunctionOpInterface funcOp) {
-    os << "function: " << funcOp.getName() << ":\n";
-    // Function arguments
-    for (BlockArgument arg : funcOp.getArguments()) {
-      LayoutInfo layout = getLayoutInfo(arg);
-      os << "argument: " << arg << "\n";
-      os << "layout  : ";
-      layout.print(os);
-      os << "\n";
-    }
-    // Function ops
-    funcOp.walk([&](Operation *op) {
-      // Skip ops that do not have results
-      if (op->getResults().empty())
-        return;
-      os << "op    : ";
-      // For control-flow ops, print the op name only.
-      if (isa<BranchOpInterface>(op) || isa<RegionBranchOpInterface>(op))
-        os << op->getName();
-      else
-        op->print(os);
-      os << "\n";
-      // Print the layout for each result.
-      for (auto [i, r] : llvm::enumerate(op->getResults())) {
-        LayoutInfo layout = getLayoutInfo(r);
-        os << "layout for result #" << i << ": ";
-        layout.print(os);
-        os << "\n";
-      }
-    });
-  };
-
-  SmallVector<FunctionOpInterface> funcOps;
-  if (auto modOp = dyn_cast<ModuleOp>(target)) {
-    for (auto funcOp : modOp.getOps<FunctionOpInterface>()) {
-      funcOps.push_back(funcOp);
-    }
-    // Collect all GpuFuncOps in the module.
-    for (auto gpuModOp : modOp.getOps<gpu::GPUModuleOp>()) {
-      for (auto gpuFuncOp : gpuModOp.getOps<FunctionOpInterface>()) {
-        funcOps.push_back(gpuFuncOp);
-      }
-    }
-  }
-  // Print the analysis result for each function.
-  for (FunctionOpInterface funcOp : funcOps) {
-    printFunctionResult(funcOp);
-  }
-}
-
-namespace {
-
-//===----------------------------------------------------------------------===//
-// LayoutAttrAssignment
-//===----------------------------------------------------------------------===//
-
-/// This class is responsible for assigning the layout attributes to the ops and
-/// their users based on the layout propagation analysis result.
-class LayoutAttrAssignment {
-public:
-  LayoutAttrAssignment(Operation *top,
-                       function_ref<LayoutInfo(Value)> getLayout)
-      : getAnalysisResult(getLayout), top(top) {}
-
-  LogicalResult run();
-
-private:
-  LogicalResult assign(Operation *op);
-  void assignToUsers(Value v, xegpu::LayoutAttr layout);
-  xegpu::LayoutAttr getLayoutAttrForValue(Value v);
-  LogicalResult resolveConflicts();
-  // Callable to get the layout of a value based on the layout propagation
-  // analysis.
-  function_ref<LayoutInfo(Value)> getAnalysisResult;
-  Operation *top;
-};
-
-} // namespace
-
-/// Helper to assign the layout attribute to the users of the value.
-void LayoutAttrAssignment::assignToUsers(Value v, xegpu::LayoutAttr layout) {
-  for (OpOperand &user : v.getUses()) {
-    Operation *owner = user.getOwner();
-    std::string attrName = xegpu::getLayoutName(user);
-    owner->setAttr(attrName, layout);
-  }
-}
-
-/// Convert the layout assigned to a value to xegpu::LayoutAttr.
-xegpu::LayoutAttr LayoutAttrAssignment::getLayoutAttrForValue(Value v) {
-  LayoutInfo layout = getAnalysisResult(v);
-  if (!layout.isAssigned())
-    return {};
-  SmallVector<int, 2> laneLayout, laneData;
-  for (auto [layout, data] : llvm::zip_equal(layout.getLayoutAsArrayRef(),
-                                             layout.getDataAsArrayRef())) {
-    laneLayout.push_back(static_cast<int>(layout));
-    laneData.push_back(static_cast<int>(data));
-  }
-  return xegpu::LayoutAttr::get(v.getContext(), laneLayout, laneData);
-}
-
-/// Assign xegpu::LayoutAttr to the op and its users. The layout is assigned
-/// based on the layout propagation analysis result.
-LogicalResult LayoutAttrAssignment::assign(Operation *op) {
-  // For function ops, propagate the function argument layout to the users.
-  if (auto func = dyn_cast<FunctionOpInterface>(op)) {
-    for (BlockArgument arg : func.getArguments()) {
-      xegpu::LayoutAttr layoutInfo = getLayoutAttrForValue(arg);
-      if (layoutInfo) {
-        assignToUsers(arg, layoutInfo);
-      }
-    }
-    return success();
-  }
-  // If no results, move on.
-  if (op->getNumResults() == 0)
-    return success();
-  // If all the results are scalars, move on.
-  if (llvm::all_of(op->getResultTypes(),
-                   [](Type t) { return t.isIntOrIndexOrFloat(); }))
-    return success();
-  // If the op has more than one result and at least one result is a tensor
-  // descriptor, exit. This case is not supported yet.
-  // TODO: Support this case.
-  if (op->getNumResults() > 1 && llvm::any_of(op->getResultTypes(), [](Type t) {
-        return isa<xegpu::TensorDescType>(t);
-      })) {
-    LLVM_DEBUG(
-        DBGS() << op->getName()
-               << " op has more than one result and at least one is a tensor "
-                  "descriptor. This case is not handled.\n");
-    return failure();
-  }
-  // If the result is a tensor descriptor, attach the layout to the tensor
-  // descriptor itself.
-  if (auto tensorDescTy =
-          dyn_cast<xegpu::TensorDescType>(op->getResultTypes()[0])) {
-    xegpu::LayoutAttr layoutInfo = getLayoutAttrForValue(op->getResult(0));
-    if (!layoutInfo) {
-      LLVM_DEBUG(DBGS() << "No layout for result of " << *op << "\n");
-      return failure();
-    }
-
-    // Clone the op, attach the layout to the result tensor descriptor, and
-    // remove the original op.
-    OpBuilder builder(op);
-    Operation *newOp = builder.clone(*op);
-    auto newTensorDescTy = xegpu::TensorDescType::get(
-        tensorDescTy.getContext(), tensorDescTy.getShape(),
-        tensorDescTy.getElementType(), tensorDescTy.getEncoding(), layoutInfo);
-    newOp->getResult(0).setType(newTensorDescTy);
-    op->replaceAllUsesWith(newOp->getResults());
-    op->erase();
-    return success();
-  }
-  // Otherwise simply attach the layout to the op itself.
-  for (auto r : op->getOpResults()) {
-    xegpu::LayoutAttr layoutInfo = getLayoutAttrForValue(r);
-    if (layoutInfo) {
-      std::string attrName = xegpu::getLayoutName(r);
-      op->setAttr(attrName, layoutInfo);
-      // Attach the layout attribute to the users of the result.
-      assignToUsers(r, layoutInfo);
-    }
-  }
-  return success();
-}
-
-/// Walk the IR and attach xegpu::LayoutAttr to all ops and their users.
-LogicalResult LayoutAttrAssignment::run() {
-  auto walkResult = top->walk([&](Operation *op) {
-    if (failed(assign(op)))
-      return WalkResult::interrupt();
-    return WalkResult::advance();
-  });
-
-  if (walkResult.wasInterrupted())
-    return failure();
-
-  return resolveConflicts();
-}
-
-/// TODO: Implement the layout conflict resolution. This must ensure mainly two
-/// things:
-/// 1) Is a given layout supported by the op? (need to query the target
-///    HW info). Otherwise can we achieve this layout using a layout conversion?
-/// 2) Do all the operands have the required layout? If not, can it
-///    be resolved using a layout conversion?
-LogicalResult LayoutAttrAssignment::resolveConflicts() { return success(); }
+static const char *const resolveSIMTTypeMismatch =
+    "resolve_simt_type_mismatch"; // Attribute name for identifying
+                                  // UnrelizedConversionCastOp added to resolve
+                                  // SIMT type mismatches.
 
 namespace {
 
@@ -867,9 +85,9 @@ getDistVecTypeBasedOnLaneLayout(xegpu::LayoutAttr layout,
   // dimensions are not distributed.
   unsigned distributionStart = originalType.getRank() - laneLayout.size();
   for (auto [i, dim] : llvm::enumerate(originalType.getShape())) {
-    if (i < distributionStart) {
+    if (i < distributionStart)
       continue;
-    }
+
     // Check if the dimension can be distributed evenly.
     if (dim % laneLayout[i - distributionStart] != 0)
       return failure();
@@ -909,6 +127,7 @@ static Value resolveDistributedTy(Value orig, T expected,
   if (isa<xegpu::TensorDescType>(orig.getType())) {
     auto castOp = rewriter.create<UnrealizedConversionCastOp>(orig.getLoc(),
                                                               expected, orig);
+    castOp->setAttr(resolveSIMTTypeMismatch, rewriter.getUnitAttr());
     return castOp.getResult(0);
   }
   llvm_unreachable("Unsupported type for reconciliation");
@@ -988,8 +207,9 @@ struct MoveFuncBodyToWarpExecuteOnLane0
         /** upperBound = **/ mlir::IntegerAttr());
     ArrayRef<Type> gpuFuncResultType = gpuFuncOp.getFunctionType().getResults();
     auto warpOp = rewriter.create<gpu::WarpExecuteOnLane0Op>(
-        laneId.getLoc(), gpuFuncResultType, laneId, subgroupSize,
-        newGpuFunc.getArguments(), newGpuFunc.getArgumentTypes());
+        laneId.getLoc(), gpuFuncResultType, laneId,
+        xegpu::targetinfo::subgroupSize, newGpuFunc.getArguments(),
+        newGpuFunc.getArgumentTypes());
     Block &warpBodyBlock = warpOp.getBodyRegion().front();
     // Replace the ReturnOp of the original gpu function with a YieldOp.
     auto origRetunOp =
@@ -1080,11 +300,14 @@ struct CreateNdDescDistribution final : public gpu::WarpDistributionPattern {
     xegpu::TensorDescType distributedTensorDescTy =
         descOp.getType().dropLayouts(); // Distributed tensor descriptor type
                                         // does not contain layout info.
-    auto newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
+    Value newDescOp = rewriter.create<xegpu::CreateNdDescOp>(
         newWarpOp.getLoc(), distributedTensorDescTy, newDescOperands,
         descOp->getAttrs());
 
     Value distributedVal = newWarpOp.getResult(operandIdx);
+    // Resolve the distributed type to the expected type.
+    newDescOp =
+        resolveDistributedTy(newDescOp, distributedVal.getType(), rewriter);
     rewriter.replaceAllUsesWith(distributedVal, newDescOp);
     return success();
   }
@@ -1485,10 +708,13 @@ struct UpdateNdOffsetDistribution final : public gpu::WarpDistributionPattern {
       }
     }
     // Create a new update op outside the warp op.
-    auto newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
+    Value newUpdateOp = rewriter.create<xegpu::UpdateNdOffsetOp>(
         newWarpOp.getLoc(), newTensorDescTy, newUpdateOperands,
         removeTemporaryLayoutAttributes(updateOp->getAttrs()));
     Value distributedVal = newWarpOp.getResult(operandIdx);
+    // Resolve the distributed type with the original type.
+    newUpdateOp =
+        resolveDistributedTy(newUpdateOp, distributedVal.getType(), rewriter);
     rewriter.replaceAllUsesWith(distributedVal, newUpdateOp);
     return success();
   }
@@ -1562,11 +788,6 @@ namespace {
 struct XeGPUSubgroupDistributePass final
     : public xegpu::impl::XeGPUSubgroupDistributeBase<
           XeGPUSubgroupDistributePass> {
-  XeGPUSubgroupDistributePass() = default;
-  XeGPUSubgroupDistributePass(const XeGPUSubgroupDistributePass &other) =
-      default;
-  XeGPUSubgroupDistributePass(xegpu::XeGPUSubgroupDistributeOptions options)
-      : XeGPUSubgroupDistributeBase(options) {}
   void runOnOperation() override;
 };
 } // namespace
@@ -1579,27 +800,29 @@ void xegpu::populateXeGPUSubgroupDistributePatterns(
 }
 
 void XeGPUSubgroupDistributePass::runOnOperation() {
-  auto &analyis = getAnalysis<RunLayoutInfoPropagation>();
-  // Print the analysis result and exit. (for testing purposes)
-  if (printOnly) {
-    auto &os = llvm::outs();
-    analyis.printAnalysisResult(os);
-    return;
-  }
-  auto getPropagatedLayout = [&](Value val) {
-    return analyis.getLayoutInfo(val);
-  };
-
-  // Assign xegpu::LayoutAttr to all ops and their users based on the layout
-  // propagation analysis result.
-  LayoutAttrAssignment layoutAssignment(getOperation(), getPropagatedLayout);
-  if (failed(layoutAssignment.run())) {
-    signalPassFailure();
-    return;
-  }
-
-  // Move all operations of a GPU function inside gpu.warp_execute_on_lane_0
-  // operation.
+  // Step 1: Attach layouts to op operands.
+  // TODO: Following assumptions are made:
+  // 1) It is assumed that there are no layout conflicts.
+  // 2) Any existing layout attributes attached to the operands are ignored.
+  Operation *op = getOperation();
+  op->walk([&](Operation *op) {
+    for (OpOperand &operand : op->getOpOperands()) {
+      // Layouts are needed for vector type only.
+      if (!isa<VectorType>(operand.get().getType()))
+        continue;
+
+      xegpu::LayoutAttr layout = xegpu::getLayoutAttr(operand);
+      if (!layout) {
+        op->emitError("Could not find layout attribute for operand ")
+            << operand.getOperandNumber() << " of operation " << op->getName();
+        signalPassFailure();
+        return;
+      }
+      xegpu::setLayoutAttr(operand, layout);
+    }
+  });
+  // Step 2: Move all operations of a GPU function inside
+  // gpu.warp_execute_on_lane_0 operation.
   {
     RewritePatternSet patterns(&getContext());
     patterns.add<MoveFuncBodyToWarpExecuteOnLane0>(&getContext());
@@ -1608,17 +831,16 @@ void XeGPUSubgroupDistributePass::runOnOperation() {
       signalPassFailure();
       return;
     }
-    // At this point, we have moved the entire function body inside the warpOp.
-    // Now move any scalar uniform code outside of the warpOp (like GPU index
-    // ops, scalar constants, etc.). This will simplify the later lowering and
-    // avoid custom patterns for these ops.
+    // At this point, we have moved the entire function body inside the
+    // warpOp. Now move any scalar uniform code outside of the warpOp (like
+    // GPU index ops, scalar constants, etc.). This will simplify the
+    // later lowering and avoid custom patterns for these ops.
     getOperation()->walk([&](Operation *op) {
-      if (auto warpOp = dyn_cast<gpu::WarpExecuteOnLane0Op>(op)) {
+      if (auto warpOp = dyn_cast<gpu::WarpExecuteOnLane0Op>(op))
         vector::moveScalarUniformCode(warpOp);
-      }
     });
   }
-  // Finally, do the SIMD to SIMT distribution.
+  // Step 3: Apply subgroup to workitem distribution patterns.
   RewritePatternSet patterns(&getContext());
   xegpu::populateXeGPUSubgroupDistributePatterns(patterns);
   // TODO: distributionFn and shuffleFn are not used at this point.
@@ -1638,4 +860,51 @@ void XeGPUSubgroupDistributePass::runOnOperation() {
     signalPassFailure();
     return;
   }
+
+  // Step 4: Finllay, clean up UnrealizedConversionCastOps that were inserted
+  // due to tensor desc type mismatches created by using upstream distribution
+  // patterns (scf.for)
+  getOperation()->walk([&](mlir::UnrealizedConversionCastOp op) {
+    // We are only interested in UnrealizedConversionCastOps there were added
+    // for resolving SIMT type mismatches.
+    if (!op->getAttr(resolveSIMTTypeMismatch))
+      return WalkResult::skip();
+
+    Value input = op.getOperand(0);
+    Value output = op.getResult(0);
+
+    // Both input and output must have tensor descriptor types.
+    xegpu::TensorDescType inputDescType =
+        mlir::dyn_cast<xegpu::TensorDescType>(input.getType());
+    xegpu::TensorDescType outputDescType =
+        mlir::dyn_cast<xegpu::TensorDescType>(output.getType());
+    assert(inputDescType && outputDescType &&
+           "Unrealized conversion cast must have tensor descriptor types");
+
+    // tensor_desc<shape, layout> -> tensor_desc<shape> Type of conversions.
+    // This occurs iside scf.for body to resolve the block argument type to
+    // SIMT type.
+    if (inputDescType.getLayout()) {
+      auto argument = mlir::dyn_cast<mlir::BlockArgument>(input);
+      if (argument) {
+        argument.setType(output.getType());
+        output.replaceAllUsesWith(argument);
+        if (auto loopOp = mlir::dyn_cast<mlir::LoopLikeOpInterface>(
+                argument.getOwner()->getParentOp())) {
+          auto result = loopOp.getTiedLoopResult(argument);
+          result.setType(output.getType());
+        }
+      }
+    }
+
+    // tensor_desc<shape> -> tensor_desc<shape, layout> Type of
+    // conversions. This occurs at the yield op of scf.for body to go back
+    // from SIMT type to original type.
+    if (outputDescType.getLayout())
+      output.replaceAllUsesWith(input);
+
+    if (op->use_empty())
+      op->erase();
+    return WalkResult::advance();
+  });
 }
diff --git a/mlir/test/Dialect/XeGPU/propagate-layout.mlir b/mlir/test/Dialect/XeGPU/propagate-layout.mlir
new file mode 100644
index 000000000000..429081079de1
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/propagate-layout.mlir
@@ -0,0 +1,430 @@
+// RUN: mlir-opt -xegpu-propagate-layout -split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: func.func @dpas_f16(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} dense<0.000000e+00> : vector<8x16xf32>
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][{{.*}}] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK: %[[T1:.*]] = xegpu.create_nd_tdesc %[[ARG1]][{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+// CHECK: %[[T2:.*]] = xegpu.load_nd %[[T0]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xf16>
+// CHECK: %[[T3:.*]] = xegpu.load_nd %[[T1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK: %[[T4:.*]] = xegpu.dpas %[[T2]], %[[T3]], %[[CST]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+// CHECK: %[[T5:.*]] = xegpu.create_nd_tdesc %[[ARG2]][{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK: xegpu.store_nd %[[T4]], %[[T5]] : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @dpas_f16(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %4 = xegpu.dpas %2, %3, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @dpas_i8(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: vector<8x32xi8>, %[[ARG1:[0-9a-zA-Z]+]]: vector<32x16xi8>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xi32>) {
+// CHECK: %[[T0:.*]] = xegpu.dpas %[[ARG0]], %[[ARG1]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16],
+func.func @dpas_i8(%arg0: vector<8x32xi8>, %arg1: vector<32x16xi8>, %arg2: memref<8x16xi32>) {
+  %c0 = arith.constant 0 : index
+  %0 = xegpu.dpas %arg0, %arg1 : vector<8x32xi8>, vector<32x16xi8> -> vector<8x16xi32>
+  %1 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xi32> -> !xegpu.tensor_desc<8x16xi32>
+  xegpu.store_nd %0, %1  : vector<8x16xi32>, !xegpu.tensor_desc<8x16xi32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @load_with_transpose_effect(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %{{.*}} = xegpu.load_nd %{{.*}} <{transpose = array<i64: 1, 0>}> {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>> -> vector<16x16xf16>
+func.func @load_with_transpose_effect(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %3 = xegpu.load_nd %1 <{transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %4 = xegpu.dpas %2, %3, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @vector_transpose(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %{{.*}} = vector.transpose %{{.*}}, [1, 0] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : vector<16x16xf16> to vector<16x16xf16>
+func.func @vector_transpose(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %4 = vector.transpose %3, [1, 0] : vector<16x16xf16> to vector<16x16xf16>
+  %5 = xegpu.dpas %2, %4, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+  %6 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %5, %6  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @extf_truncf(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>, %[[ARG1:[0-9a-zA-Z]+]]:
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>) -> vector<8x16xf32> {
+// CHECK: %[[T2:.*]] = arith.extf %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : vector<16x16xf16> to vector<16x16xf32>
+// CHECK-NEXT: %{{.*}} = arith.truncf %[[T2]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : vector<16x16xf32> to vector<16x16xf16>
+func.func @extf_truncf(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>) -> vector<8x16xf32> {
+  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %2 = arith.extf %1 : vector<16x16xf16> to vector<16x16xf32>
+  %3 = arith.truncf %2 : vector<16x16xf32> to vector<16x16xf16>
+  %4 = xegpu.dpas %0, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  return %4 : vector<8x16xf32>
+}
+
+// -----
+// CHECK-LABEL: func.func @load_gather_with_transpose_effect(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<256xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>}
+// CHECK-SAME:  dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+// CHECK-NEXT: %[[CST0:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} dense<true> : vector<16xi1>
+// CHECK-NEXT: %[[T2:.*]] = xegpu.create_tdesc %[[ARG1]], %[[CST]] : memref<256xf16>, vector<16xindex> ->
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>>
+// CHECK-NEXT: %{{.*}} = xegpu.load %[[T2]], %[[CST0]] <{transpose}> {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 2]>>, vector<16xi1> -> vector<16x16xf16>
+func.func @load_gather_with_transpose_effect(%arg0: memref<8x16xf16>, %arg1: memref<256xf16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+  %cst_0 = arith.constant dense<true> : vector<16xi1>
+  %2 = xegpu.create_tdesc %arg1, %cst : memref<256xf16>, vector<16xindex> -> !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>
+  %3 = xegpu.load %2, %cst_0 <{transpose}> : !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>, vector<16xi1> -> vector<16x16xf16>
+  %4 = xegpu.dpas %1, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @load_gather_1d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf32>, %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>) {
+// CHECK: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>}
+// CHECK-SAME: dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+// CHECK-NEXT: %[[CST0:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} dense<true> : vector<16xi1>
+// CHECK-NEXT: %[[T0:.*]] = xegpu.create_tdesc %[[ARG0]], %[[CST]] : memref<256xf32>, vector<16xindex> ->
+// CHECK-SAME: !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+// CHECK-NEXT: %{{.*}} = xegpu.load %[[T0]], %[[CST0]]  {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>, #xegpu.layout<lane_layout = [16], lane_data = [1]>>, vector<16xi1> -> vector<16xf32>
+func.func @load_gather_1d(%arg0: memref<256xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
+  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+  %cst_0 = arith.constant dense<true> : vector<16xi1>
+  %0 = xegpu.create_tdesc %arg0, %cst : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+  %1 = xegpu.load %0, %cst_0  : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
+  xegpu.store_nd %1, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @store_scatter_with_transpose_effect(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<128xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_tdesc %[[ARG0]], %{{.*}} : memref<128xf32>, vector<16xindex> ->
+// CHECK-SAME: !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>, #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>
+// CHECK-NEXT: xegpu.store %{{.*}}, %[[T0]], %{{.*}} <{transpose}> : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>,
+// CHECK-SAME: #xegpu.layout<lane_layout = [16, 1], lane_data = [1, 1]>>, vector<16xi1>
+func.func @store_scatter_with_transpose_effect(%arg0: memref<128xf32>) {
+  %cst = arith.constant dense<1.000000e+00> : vector<8x16xf32>
+  %cst_0 = arith.constant dense<true> : vector<16xi1>
+  %cst_1 = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+  %0 = xegpu.create_tdesc %arg0, %cst_1 : memref<128xf32>, vector<16xindex> -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
+  xegpu.store %cst, %0, %cst_0 <{transpose}> : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<16xi1>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @store_scatter_1d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: vector<16xf32>, %[[ARG1:[0-9a-zA-Z]+]]: memref<256xf32>) {
+// CHECK: xegpu.store %[[ARG0]], %{{.*}}, %{{.*}}  : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>,
+// CHECK-SAME: #xegpu.layout<lane_layout = [16], lane_data = [1]>>, vector<16xi1>
+func.func @store_scatter_1d(%arg0: vector<16xf32>, %arg1: memref<256xf32>) {
+  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
+  %cst_0 = arith.constant dense<true> : vector<16xi1>
+  %0 = xegpu.create_tdesc %arg1, %cst : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
+  xegpu.store %arg0, %0, %cst_0  : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @vector_bitcast_i16_to_f16(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xi16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xi16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %{{.*}} = vector.bitcast %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xi16> to vector<8x16xf16>
+// CHECK: %{{.*}} = vector.bitcast %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : vector<16x16xi16> to vector<16x16xf16>
+func.func @vector_bitcast_i16_to_f16(%arg0: memref<8x16xi16>, %arg1: memref<16x16xi16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xi16> -> !xegpu.tensor_desc<8x16xi16>
+  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xi16> -> !xegpu.tensor_desc<16x16xi16>
+  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xi16> -> vector<8x16xi16>
+  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x16xi16> -> vector<16x16xi16>
+  %4 = vector.bitcast %2 : vector<8x16xi16> to vector<8x16xf16>
+  %5 = vector.bitcast %3 : vector<16x16xi16> to vector<16x16xf16>
+  %6 = xegpu.dpas %4, %5 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  %7 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %6, %7  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @binary_op_one_use(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>,
+// CHECK-SAME: %[[ARG2:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>) {
+// CHECK: %[[T1:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT: %[[T2:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT: %{{.*}} = arith.addf %[[T1]], %[[T2]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : vector<16x16xf16>
+func.func @binary_op_one_use(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: !xegpu.tensor_desc<8x16xf32>) {
+  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %2 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %3 = arith.addf %1, %2 : vector<16x16xf16>
+  %4 = xegpu.dpas %0, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  xegpu.store_nd %4, %arg2  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @binary_op_multiple_uses(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG2:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG3:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>) {
+// CHECK: %[[T2:.*]] = arith.addf %{{.*}}, %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<16x16xf16>
+// CHECK: %[[T3:.*]] = xegpu.dpas %{{.*}}, %[[T2]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+// CHECK-NEXT: xegpu.store_nd %[[T3]], %[[ARG2]]  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT: xegpu.store_nd %[[T2]], %[[ARG3]]  : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @binary_op_multiple_uses(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: !xegpu.tensor_desc<8x16xf32>, %arg3: !xegpu.tensor_desc<16x16xf16>) {
+  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+  %cst = arith.constant dense<1.000000e+00> : vector<16x16xf16>
+  %2 = arith.addf %1, %cst : vector<16x16xf16>
+  %3 = xegpu.dpas %0, %2 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  xegpu.store_nd %3, %arg2  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %2, %arg3  : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @for_op(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<8x128xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<128x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<8x128xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT: %[[T1:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<128x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+// CHECK-NEXT: %[[CST:.*]] = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} dense<0.000000e+00> : vector<8x16xf32>
+// CHECK-NEXT: %[[T2:.*]]:3 = scf.for %{{.*}} iter_args(%[[ARG4:.*]] = %[[T0]], %[[ARG5:.*]] = %[[T1]], %[[ARG6:.*]] = %[[CST]]) ->
+// CHECK-SAME: (!xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>, vector<8x16xf32>) {
+// CHECK-NEXT:   %[[T4:.*]] = xegpu.load_nd %[[ARG4]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xf16>
+// CHECK-NEXT:   %[[T5:.*]] = xegpu.load_nd %[[ARG5]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT:   %[[T6:.*]] = xegpu.dpas %[[T4]], %[[T5]], %[[ARG6]] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+// CHECK-NEXT:   %[[T7:.*]] = xegpu.update_nd_offset %[[ARG4]], [{{.*}}] : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT:   %[[T8:.*]] = xegpu.update_nd_offset %[[ARG5]], [{{.*}}] : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+// CHECK-NEXT:   scf.yield %[[T7]], %[[T8]], %[[T6]] : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>, vector<8x16xf32>
+// CHECK-NEXT: } {layout_result_2 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
+// CHECK-NEXT: %[[T3:.*]] = xegpu.create_nd_tdesc %[[ARG2]][{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT: xegpu.store_nd %[[T2]]#2, %[[T3]] : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @for_op(%arg0: memref<8x128xf16>, %arg1: memref<128x16xf16>, %arg2: memref<8x16xf32>) {
+  %c0 = arith.constant 0 : index
+  %c128 = arith.constant 128 : index
+  %c16 = arith.constant 16 : index
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x128xf16> -> !xegpu.tensor_desc<8x16xf16>
+  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<128x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
+  %2:3 = scf.for %arg3 = %c0 to %c128 step %c16 iter_args(%arg4 = %0, %arg5 = %1, %arg6 = %cst) -> (!xegpu.tensor_desc<8x16xf16>, !xegpu.tensor_desc<16x16xf16>, vector<8x16xf32>) {
+    %4 = xegpu.load_nd %arg4  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+    %5 = xegpu.load_nd %arg5  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+    %6 = xegpu.dpas %4, %5, %arg6 : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
+    %7 = xegpu.update_nd_offset %arg4, [%c0, %c16] : !xegpu.tensor_desc<8x16xf16>
+    %8 = xegpu.update_nd_offset %arg5, [%c16, %c0] : !xegpu.tensor_desc<16x16xf16>
+    scf.yield %7, %8, %6 : !xegpu.tensor_desc<8x16xf16>, !xegpu.tensor_desc<16x16xf16>, vector<8x16xf32>
+  }
+  %3 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %2#2, %3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @if_single_use(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>,
+// CHECK-SAME: %[[ARG2:[0-9a-zA-Z]+]]: i1, %[[ARG3:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>) {
+// CHECK:  %{{.*}} = scf.if %[[ARG2]] -> (vector<16x16xf16>) {
+// CHECK-NEXT:    %[[T3:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT:    scf.yield %[[T3]] : vector<16x16xf16>
+// CHECK-NEXT:  } else {
+// CHECK-NEXT:    %[[T4:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT:    scf.yield %[[T4]] : vector<16x16xf16>
+// CHECK-NEXT:  } {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>}
+func.func @if_single_use(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: i1, %arg3: !xegpu.tensor_desc<8x16xf32>) {
+  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %1 = scf.if %arg2 -> (vector<16x16xf16>) {
+    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+    scf.yield %3 : vector<16x16xf16>
+  } else {
+    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+    scf.yield %3 : vector<16x16xf16>
+  }
+  %2 = xegpu.dpas %0, %1 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  xegpu.store_nd %2, %arg3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @if_multiple_uses(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG2:[0-9a-zA-Z]+]]: i1, %[[ARG3:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>,
+// CHECK-SAME: %[[ARG4:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>) {
+// CHECK: %[[T1:.*]] = scf.if %[[ARG2]] -> (vector<16x16xf16>) {
+// CHECK-NEXT:       %[[T3:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT:       scf.yield %[[T3]] : vector<16x16xf16>
+// CHECK-NEXT:     } else {
+// CHECK-NEXT:       %[[T4:.*]] = xegpu.load_nd %[[ARG1]]  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} :
+// CHECK-SAME: !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<16x16xf16>
+// CHECK-NEXT:       scf.yield %[[T4]] : vector<16x16xf16>
+// CHECK-NEXT:     } {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
+func.func @if_multiple_uses(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: i1, %arg3: !xegpu.tensor_desc<8x16xf32>, %arg4: !xegpu.tensor_desc<16x16xf16>) {
+  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
+  %1 = scf.if %arg2 -> (vector<16x16xf16>) {
+    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+    scf.yield %3 : vector<16x16xf16>
+  } else {
+    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
+    scf.yield %3 : vector<16x16xf16>
+  }
+  %2 = xegpu.dpas %0, %1 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+  xegpu.store_nd %2, %arg3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
+  xegpu.store_nd %1, %arg4  : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @vector_outer_reduction(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: vector<16x16xf32>, %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>) {
+// CHECK: %{{.*}} = vector.multi_reduction <add>, %[[ARG0]], %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} [0] : vector<16x16xf32> to vector<16xf32>
+func.func @vector_outer_reduction(%arg0: vector<16x16xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
+  %cst = arith.constant dense<0.000000e+00> : vector<16xf32>
+  %0 = vector.multi_reduction <add>, %arg0, %cst [0] : vector<16x16xf32> to vector<16xf32>
+  xegpu.store_nd %0, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @vector_inner_reduction(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: vector<16x16xf32>, %[[ARG1:[0-9a-zA-Z]+]]: !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>) {
+// CHECK: %{{.*}} = vector.multi_reduction <add>, %[[ARG0]], %{{.*}} {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} [1] : vector<16x16xf32> to vector<16xf32>
+func.func @vector_inner_reduction(%arg0: vector<16x16xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
+  %cst = arith.constant dense<0.000000e+00> : vector<16xf32>
+  %0 = vector.multi_reduction <add>, %arg0, %cst [1] : vector<16x16xf32> to vector<16xf32>
+  xegpu.store_nd %0, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @update_nd_offset_1d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf32> -> !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+// CHECK-NEXT: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%{{.*}}] : !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+func.func @update_nd_offset_1d(%arg0: memref<256xf32>){
+  %c0 = arith.constant 0 : index
+  %c32 = arith.constant 32 : index
+  %1 = arith.constant dense<1.000000e+00> : vector<16xf32>
+  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
+  %2 = xegpu.update_nd_offset %0, [%c32] : !xegpu.tensor_desc<16xf32>
+  xegpu.store_nd %1, %2 : vector<16xf32>, !xegpu.tensor_desc<16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @update_nd_offset_2d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}, %{{.*}}] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%{{.*}}, %{{.*}}] : !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @update_nd_offset_2d(%arg0: memref<256x256xf32>){
+  %c0 = arith.constant 0 : index
+  %c32 = arith.constant 32 : index
+  %1 = arith.constant dense<1.000000e+00> : vector<16x16xf32>
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
+  %2 = xegpu.update_nd_offset %0, [%c32, %c32] : !xegpu.tensor_desc<16x16xf32>
+  xegpu.store_nd %1, %2 : vector<16x16xf32>, !xegpu.tensor_desc<16x16xf32>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @prefetch_2d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}, %{{.*}}] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+// CHECK-NEXT: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+func.func @prefetch_2d(%arg0: memref<256x256xf16>){
+  %c0 = arith.constant 0 : index
+  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
+  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16x16xf16>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @prefetch_1d(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+// CHECK-NEXT: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+func.func @prefetch_1d(%arg0: memref<256xf16>){
+  %c0 = arith.constant 0 : index
+  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
+  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16xf16>
+  return
+}
+
+// -----
+// CHECK-LABEL: func.func @test_scf_while_and_condition(
+// CHECK-SAME: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf32>, %[[ARG1:[0-9a-zA-Z]+]]: memref<256xf32>) {
+// CHECK: %{{.*}}:3 = scf.while ({{.*}}) : (vector<16xf32>, i32, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>)
+// CHECK-SAME: -> (vector<16xf32>, i32, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>) {
+// CHECK:       scf.condition(%{{.*}}) {{.*}} : vector<16xf32>, i32, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+// CHECK-NEXT: } do {
+// CHECK-NEXT: ^bb0(%{{.*}}: vector<16xf32>, %{{.*}}: i32, %{{.*}}: !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>):
+// CHECK:     scf.yield {{.*}} : vector<16xf32>, i32, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+// CHECK-NEXT: } attributes {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>}
+func.func @test_scf_while_and_condition(%arg0: memref<256xf32>, %arg1: memref<256xf32>) {
+  %c0 = arith.constant 0 : i32
+  %c16 = arith.constant 16 : i32
+  %c256 = arith.constant 256 : i32
+  %0 = xegpu.create_nd_tdesc %arg0[0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
+  %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<16xf32> -> vector<16xf32>
+  %2 = xegpu.create_nd_tdesc %arg1[0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
+
+  %3:3 = scf.while (%arg2 = %1, %arg3 = %c0, %arg4 = %0) : (vector<16xf32>, i32, !xegpu.tensor_desc<16xf32>)
+    -> (vector<16xf32>, i32, !xegpu.tensor_desc<16xf32>) {
+    %4 = arith.cmpi slt, %arg3, %c256 : i32
+    scf.condition(%4) %arg2, %arg3, %arg4 : vector<16xf32>, i32, !xegpu.tensor_desc<16xf32>
+  } do {
+  ^bb0(%arg2: vector<16xf32>, %arg3: i32, %arg4: !xegpu.tensor_desc<16xf32>):
+    xegpu.store_nd %arg2, %2  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
+    %4 = arith.addi %arg3, %c16 : i32
+    %5 = xegpu.update_nd_offset %arg4, [16] : !xegpu.tensor_desc<16xf32>
+    %6 = xegpu.load_nd %5  : !xegpu.tensor_desc<16xf32> -> vector<16xf32>
+    scf.yield %6, %4, %5 : vector<16xf32>, i32, !xegpu.tensor_desc<16xf32>
+  }
+  return
+}
diff --git a/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir b/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
new file mode 100644
index 000000000000..a59633b0cbd9
--- /dev/null
+++ b/mlir/test/Dialect/XeGPU/subgroup-distribute.mlir
@@ -0,0 +1,280 @@
+// RUN: mlir-opt -xegpu-subgroup-distribute -canonicalize -cse -split-input-file %s | FileCheck %s
+
+// CHECK-LABEL: gpu.func @store_nd_1d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16xf32>) {
+// CHECK-DAG: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<1xf32>
+// CHECK-DAG: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
+// CHECK: xegpu.store_nd %[[CST]], %[[T0]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
+// CHECK: gpu.return
+gpu.module @test {
+  gpu.func @store_nd_1d(%arg0: memref<16xf32>) {
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} dense<1.000000e+00> : vector<16xf32>
+    %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    xegpu.store_nd %cst, %0  : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @store_nd_2d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
+// CHECK-DAG: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16xf16>
+// CHECK-DAG: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: xegpu.store_nd %[[CST]], %[[T0]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
+gpu.module @test {
+  gpu.func @store_nd_2d(%arg0: memref<16x16xf16>) {
+    %c0 = arith.constant 0 : index
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} dense<1.000000e+00> : vector<16x16xf16>
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %cst, %0 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+
+
+// -----
+// CHECK-LABEL: gpu.func @load_nd_1d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16xf32>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
+// CHECK-DAG: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16xf32> -> vector<1xf32>
+// CHECK-DAG: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
+// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
+gpu.module @test {
+  gpu.func @load_nd_1d(%arg0: memref<16xf32>, %arg1: memref<16xf32>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} : !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>> -> vector<16xf32>
+    %2 = xegpu.create_nd_tdesc %arg1[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    xegpu.store_nd %1, %2 : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @load_nd_2d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK-DAG: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
+// CHECK-DAG: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
+gpu.module @test {
+  gpu.func @load_nd_2d(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf16>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<16x16xf16>
+    %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %1, %2 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @load_nd_array_length
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>>
+// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>> -> vector<32xf16>
+// CHECK: %[[T2:.*]] = vector.shape_cast %[[T1]] : vector<32xf16> to vector<2x16x1xf16>
+// CHECK: %[[T3:.*]] = vector.extract %[[T2]][0] : vector<16x1xf16> from vector<2x16x1xf16>
+// CHECK-DAG: %[[T4:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK-DAG: %[[T5:.*]] = vector.shape_cast %[[T3]] : vector<16x1xf16> to vector<16xf16>
+// CHECK: xegpu.store_nd %[[T5]], %[[T4]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
+gpu.module @test {
+  gpu.func @load_nd_array_length(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf16>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<2x16x16xf16>
+    %2 = vector.extract %1[%c0] {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<16x16xf16> from vector<2x16x16xf16>
+    %3 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %2, %3 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @load_dpas_store
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]] <{packed}> : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
+// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+// CHECK: %[[T3:.*]] = xegpu.load_nd %[[T2]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8xf16>
+// CHECK-DAG: %[[T4:.*]] = xegpu.dpas %[[T3]], %[[T1]] : vector<8xf16>, vector<16xf16> -> vector<8xf32>
+// CHECK-DAG: %[[T5:.*]] = xegpu.create_nd_tdesc %[[ARG2]][%{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+// CHECK: xegpu.store_nd %[[T4]], %[[T5]]  : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
+gpu.module @test {
+  gpu.func @load_dpas_store(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xf16>
+    %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+    %3 = xegpu.load_nd %2  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+    %4 = xegpu.dpas %1, %3 {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+    %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+
+// -----
+// CHECK-LABEL: gpu.func @load_dpas_postop_store
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]] <{packed}> : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
+// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
+// CHECK: %[[T3:.*]] = xegpu.load_nd %[[T2]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8xf16>
+// CHECK-DAG: %[[T4:.*]] = xegpu.dpas %[[T3]], %[[T1]] : vector<8xf16>, vector<16xf16> -> vector<8xf32>
+// CHECK: %[[T5:.*]] = vector.shape_cast %[[T4]] : vector<8xf32> to vector<8x1xf32>
+// CHECK: %[[T6:.*]] = math.exp %[[T5]] {{{.*}}} : vector<8x1xf32>
+// CHECK-DAG: %[[T8:.*]] = vector.shape_cast %[[T6]] : vector<8x1xf32> to vector<8xf32>
+// CHECK-DAG: %[[T7:.*]] = xegpu.create_nd_tdesc %[[ARG2]][{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
+// CHECK: xegpu.store_nd %[[T8]], %[[T7]]  : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
+gpu.module @test {
+  gpu.func @load_dpas_postop_store(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<8x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xf16>
+    %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+    %3 = xegpu.load_nd %2  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xf16>
+    %4 = xegpu.dpas %1, %3 {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
+    %5 = math.exp %4 {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xf32>
+    %6 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %5, %6 : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @create_nd_tdesc_non_memref
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: ui64, %[[ARG1:[0-9a-zA-Z]+]]: ui64, %[[ARG2:[0-9a-zA-Z]+]]: index,
+// CHECK-SAME: %[[ARG3:[0-9a-zA-Z]+]]: index, %[[ARG4:[0-9a-zA-Z]+]]: index,
+// CHECK-SAME: %[[ARG5:[0-9a-zA-Z]+]]: index, %[[ARG6:[0-9a-zA-Z]+]]: index, %[[ARG7:[0-9a-zA-Z]+]]: index) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][{{.*}}], [%[[ARG2]], %[[ARG3]]], [%[[ARG4]], %[[ARG5]]] : ui64 -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
+// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][{{.*}}], [%[[ARG2]], %[[ARG3]]], [%[[ARG4]], %[[ARG5]]] : ui64 -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
+gpu.module @test {
+  gpu.func @create_nd_tdesc_non_memref(%arg0: ui64, %arg1: ui64, %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0], [%arg2, %arg3], [%arg4, %arg5] : ui64 -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.load_nd %0  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<16x16xf16>
+    %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0], [%arg2, %arg3], [%arg4, %arg5] : ui64 -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %1, %2 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// TODO: gemm does not use update_nd_offset because of an issue in scf-for distribution.
+// CHECK-LABEL: gpu.func @gemm
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<1024x1024xbf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<1024x1024xbf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<1024x1024xf32>) {
+// CHECK-DAG: %[[BLOCK_ID_X:.*]] = gpu.block_id x
+// CHECK-DAG: %[[BLOCK_ID_Y:.*]] = gpu.block_id y
+// CHECK-DAG: %[[Y_COORD:.*]] = arith.muli %[[BLOCK_ID_Y]], %c16 : index
+// CHECK-DAG: %[[X_COORD:.*]] = arith.muli %[[BLOCK_ID_X]], %c8 : index
+// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG2]][%[[X_COORD]], %[[Y_COORD]]] : memref<1024x1024xf32> -> !xegpu.tensor_desc<8x16xf32>
+// CHECK-NEXT: %[[T3:.*]] = xegpu.load_nd %[[T2]] : !xegpu.tensor_desc<8x16xf32> -> vector<8xf32>
+// CHECK-NEXT: %[[T4:.*]] = vector.shape_cast %[[T3]] : vector<8xf32> to vector<8x1xf32>
+// CHECK: %[[T5:.*]] = scf.for %[[K:.*]] = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[ARG4:.*]] = %[[T4]]) -> (vector<8x1xf32>) {
+// CHECK-DAG: %[[T10:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%[[K]], %[[Y_COORD]]] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<16x16xbf16>
+// CHECK-DAG: %[[T11:.*]] = xegpu.load_nd %[[T10]] <{packed}> : !xegpu.tensor_desc<16x16xbf16> -> vector<16xbf16>
+// CHECK-DAG: %[[T12:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%[[X_COORD]], %[[K]]] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<8x16xbf16>
+// CHECK-DAG: %[[T13:.*]] = xegpu.load_nd %[[T12]] : !xegpu.tensor_desc<8x16xbf16> -> vector<8xbf16>
+// CHECK-DAG: %[[T14:.*]] = vector.shape_cast %[[ARG4]] : vector<8x1xf32> to vector<8xf32>
+// CHECK-NEXT: %[[T15:.*]] = xegpu.dpas %[[T13]], %[[T11]], %[[T14]] : vector<8xbf16>, vector<16xbf16>, vector<8xf32> -> vector<8xf32>
+// CHECK-NEXT: %[[T16:.*]] = vector.shape_cast %[[T15]] : vector<8xf32> to vector<8x1xf32>
+// CHECK-NEXT: scf.yield %[[T16]] : vector<8x1xf32>
+// CHECK-NEXT: }
+// CHECK-NEXT: %[[T9:.*]] = vector.shape_cast %[[T5]] : vector<8x1xf32> to vector<8xf32>
+// CHECK-NEXT: xegpu.store_nd %[[T9]], %[[T2]] : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
+gpu.module @test {
+gpu.func @gemm(%arg0: memref<1024x1024xbf16>, %arg1: memref<1024x1024xbf16>, %arg2: memref<1024x1024xf32>){
+  %c0 = arith.constant 0 : index
+  %c16 = arith.constant 16 : index
+  %c8 = arith.constant 8 : index
+  %c1024 = arith.constant 1024 : index
+  %block_id_x = gpu.block_id  x
+  %block_id_y = gpu.block_id  y
+  %0 = arith.muli %block_id_x, %c8 : index
+  %1 = arith.muli %block_id_y, %c16 : index
+  %2 = xegpu.create_nd_tdesc %arg2[%0, %1] : memref<1024x1024xf32> -> !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+  %3 = xegpu.load_nd %2  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xf32>
+  %4 = scf.for %arg3 = %c0 to %c1024 step %c16 iter_args(%arg4 = %3) -> (vector<8x16xf32>) {
+    %5 = xegpu.create_nd_tdesc %arg0[%0, %arg3] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<8x16xbf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %6 = xegpu.create_nd_tdesc %arg1[%arg3, %1] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<16x16xbf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>>
+    %7 = xegpu.load_nd %5  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : !xegpu.tensor_desc<8x16xbf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>> -> vector<8x16xbf16>
+    %8 = xegpu.load_nd %6  {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>} : !xegpu.tensor_desc<16x16xbf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [2, 1]>> -> vector<16x16xbf16>
+    %9 = xegpu.dpas %7, %8, %arg4 {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} : vector<8x16xbf16>, vector<16x16xbf16>, vector<8x16xf32> -> vector<8x16xf32>
+    scf.yield %9 : vector<8x16xf32>
+  } {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>}
+  xegpu.store_nd %4, %2 : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+  gpu.return
+}
+}
+
+// -----
+// CHECK-LABEL: gpu.func @update_nd_offset_1d(
+// CHECK: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf32>) {
+// CHECK: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<1xf32>
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
+// CHECK: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%c32] : !xegpu.tensor_desc<16xf32>
+// CHECK: xegpu.store_nd %[[CST]], %[[T1]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
+gpu.module @test {
+  gpu.func @update_nd_offset_1d(%arg0: memref<256xf32>) {
+    %c0 = arith.constant 0 : index
+    %c32 = arith.constant 32 : index
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [16], lane_data = [1]>} dense<1.000000e+00> : vector<16xf32>
+    %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    %1 = xegpu.update_nd_offset %0, [%c32] : !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    xegpu.store_nd %cst, %1  : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @update_nd_offset_2d
+// CHECK: %[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf32>) {
+// CHECK: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16xf32>
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
+// CHECK: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%c32, %c32] : !xegpu.tensor_desc<16x16xf32>
+// CHECK: xegpu.store_nd %[[CST]], %[[T1]]  : vector<16xf32>, !xegpu.tensor_desc<16x16xf32>
+gpu.module @test {
+  gpu.func @update_nd_offset_2d(%arg0: memref<256x256xf32>) {
+    %c0 = arith.constant 0 : index
+    %c32 = arith.constant 32 : index
+    %cst = arith.constant {layout_result_0 = #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>} dense<1.000000e+00> : vector<16x16xf32>
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    %1 = xegpu.update_nd_offset %0, [%c32, %c32] : !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.store_nd %cst, %1  : vector<16x16xf32>, !xegpu.tensor_desc<16x16xf32, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @prefetch_2d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
+// CHECK: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16>
+gpu.module @test {
+  gpu.func @prefetch_2d(%arg0: memref<256x256xf16>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16, #xegpu.layout<lane_layout = [1, 16], lane_data = [1, 1]>>
+    gpu.return
+  }
+}
+
+// -----
+// CHECK-LABEL: gpu.func @prefetch_1d
+// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<256xf16>) {
+// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
+// CHECK: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16>
+gpu.module @test {
+  gpu.func @prefetch_1d(%arg0: memref<256xf16>) {
+    %c0 = arith.constant 0 : index
+    %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16, #xegpu.layout<lane_layout = [16], lane_data = [1]>>
+    gpu.return
+  }
+}
diff --git a/mlir/test/Dialect/XeGPU/subgroup-distribution.mlir b/mlir/test/Dialect/XeGPU/subgroup-distribution.mlir
deleted file mode 100644
index e5606c564250..000000000000
--- a/mlir/test/Dialect/XeGPU/subgroup-distribution.mlir
+++ /dev/null
@@ -1,275 +0,0 @@
-// RUN: mlir-opt -xegpu-subgroup-distribute -cse -split-input-file %s | FileCheck %s
-
-// CHECK-LABEL: gpu.func @store_nd_1d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16xf32>) {
-// CHECK-DAG: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<1xf32>
-// CHECK-DAG: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-// CHECK: xegpu.store_nd %[[CST]], %[[T0]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
-// CHECK: gpu.return
-gpu.module @test {
-gpu.func @store_nd_1d(%arg0: memref<16xf32>){
-  %c0 = arith.constant 0 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-  xegpu.store_nd %1, %0 : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @store_nd_2d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
-// CHECK-DAG: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16xf16>
-// CHECK-DAG: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: xegpu.store_nd %[[CST]], %[[T0]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
-gpu.module @test {
-gpu.func @store_nd_2d(%arg0: memref<16x16xf16>){
-  %c0 = arith.constant 0 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16x16xf16>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.store_nd %1, %0 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  gpu.return
-}
-}
-
-
-
-// -----
-// CHECK-LABEL: gpu.func @load_nd_1d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16xf32>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16xf32>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-// CHECK-DAG: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16xf32> -> vector<1xf32>
-// CHECK-DAG: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
-gpu.module @test {
-gpu.func @load_nd_1d(%arg0: memref<16xf32>, %arg1: memref<16xf32>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-  %1 = xegpu.load_nd %0 : !xegpu.tensor_desc<16xf32> -> vector<16xf32>
-  %2 = xegpu.create_nd_tdesc %arg1[%c0] : memref<16xf32> -> !xegpu.tensor_desc<16xf32>
-  xegpu.store_nd %1, %2 : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @load_nd_2d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-DAG: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
-// CHECK-DAG: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
-gpu.module @test {
-gpu.func @load_nd_2d(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %1 = xegpu.load_nd %0 : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.store_nd %1, %2 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @load_nd_array_length
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>>
-// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>> -> vector<32xf16>
-// CHECK: %[[T2:.*]] = vector.shape_cast %[[T1]] : vector<32xf16> to vector<2x16x1xf16>
-// CHECK: %[[T3:.*]] = vector.extract %[[T2]][0] : vector<16x1xf16> from vector<2x16x1xf16>
-// CHECK-DAG: %[[T4:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-DAG: %[[T5:.*]] = vector.shape_cast %[[T3]] : vector<16x1xf16> to vector<16xf16>
-// CHECK: xegpu.store_nd %[[T5]], %[[T4]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
-gpu.module @test {
-gpu.func @load_nd_array_length(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>>
-  %1 = xegpu.load_nd %0 : !xegpu.tensor_desc<16x16xf16, #xegpu.block_tdesc_attr<array_length = 2 : i64>> -> vector<2x16x16xf16>
-  %2 = vector.extract %1[%c0] : vector<16x16xf16> from vector<2x16x16xf16>
-  %3 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.store_nd %2, %3 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @dpas
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: vector<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: vector<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: vector<8x16xf32>, %[[ARG3:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
-// CHECK: %[[T1:.*]]:3 = gpu.warp_execute_on_lane_0(%{{.*}})[16] args(%[[ARG0]], %[[ARG1]], %[[ARG2]], %[[ARG3]]
-// CHECK-SAME: vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32>, memref<8x16xf32>) -> (vector<8x1xf16>, vector<16x1xf16>, vector<8x1xf32>) {
-// CHECK: ^bb0(%[[ARG4:[0-9a-zA-Z]+]]: vector<8x16xf16>, %[[ARG5:[0-9a-zA-Z]+]]: vector<16x16xf16>, %[[ARG6:[0-9a-zA-Z]+]]: vector<8x16xf32>, %[[ARG7:[0-9a-zA-Z]+]]: memref<8x16xf32>):
-// CHECK:  gpu.yield %[[ARG4]], %[[ARG5]], %[[ARG6]] : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32>
-// CHECK: }
-// CHECK-DAG: %[[T2:.*]] = vector.shape_cast %[[T1]]#0 : vector<8x1xf16> to vector<8xf16>
-// CHECK-DAG: %[[T3:.*]] = vector.shape_cast %[[T1]]#1 : vector<16x1xf16> to vector<16xf16>
-// CHECK-DAG: %[[T4:.*]] = vector.shape_cast %[[T1]]#2 : vector<8x1xf32> to vector<8xf32>
-// CHECK: %[[T5:.*]] = xegpu.dpas %[[T2]], %[[T3]], %[[T4]] : vector<8xf16>, vector<16xf16>, vector<8xf32> -> vector<8xf32>
-// CHECK: %[[T6:.*]] = xegpu.create_nd_tdesc %[[ARG3]][%{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK: xegpu.store_nd %[[T5]], %[[T6]]  : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
-gpu.module @test {
-gpu.func @dpas(%arg0: vector<8x16xf16>, %arg1: vector<16x16xf16>, %arg3: vector<8x16xf32>, %arg2: memref<8x16xf32>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.dpas %arg0, %arg1, %arg3 : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-  %3 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %0, %3 : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @load_dpas_store
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<8x16xf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<16x16xf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<8x16xf32>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%{{.*}}] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]] <{packed}> : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
-// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK: %[[T3:.*]] = xegpu.load_nd %[[T2]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8xf16>
-// CHECK-DAG: %[[T4:.*]] = xegpu.dpas %[[T3]], %[[T1]] : vector<8xf16>, vector<16xf16> -> vector<8xf32>
-// CHECK-DAG: %[[T5:.*]] = xegpu.create_nd_tdesc %[[ARG2]][%{{.*}}] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK: xegpu.store_nd %[[T4]], %[[T5]]  : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
-gpu.module @test {
-gpu.func @load_dpas_store(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg3: memref<8x16xf32>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.load_nd %0 : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %3 = xegpu.load_nd %2 : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %4 = xegpu.dpas %1, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  %5 = xegpu.create_nd_tdesc %arg3[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %4, %5 : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  gpu.return
-}
-}
-
-// -----
-gpu.module @test {
-// CHECK-LABEL: gpu.func @create_nd_tdesc_non_memref
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: ui64, %[[ARG1:[0-9a-zA-Z]+]]: ui64, %[[ARG2:[0-9a-zA-Z]+]]: index,
-// CHECK-SAME: %[[ARG3:[0-9a-zA-Z]+]]: index, %[[ARG4:[0-9a-zA-Z]+]]: index,
-// CHECK-SAME: %[[ARG5:[0-9a-zA-Z]+]]: index, %[[ARG6:[0-9a-zA-Z]+]]: index, %[[ARG7:[0-9a-zA-Z]+]]: index) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][{{.*}}], [%[[ARG2]], %[[ARG3]]], [%[[ARG4]], %[[ARG5]]] : ui64 -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16xf16>
-// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG1]][{{.*}}], [%[[ARG2]], %[[ARG3]]], [%[[ARG4]], %[[ARG5]]] : ui64 -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: xegpu.store_nd %[[T1]], %[[T2]]  : vector<16xf16>, !xegpu.tensor_desc<16x16xf16>
-gpu.func @create_nd_tdesc_non_memref(%arg0: ui64, %arg1: ui64,
-  %arg2: index, %arg3: index, %arg4: index, %arg5: index, %arg6: index, %arg7: index) {
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0 [%c0, %c0], [%arg2, %arg3], [%arg4, %arg5] : ui64 -> !xegpu.tensor_desc<16x16xf16>
-  %1 = xegpu.load_nd %0 : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %2 = xegpu.create_nd_tdesc %arg1[%c0, %c0], [%arg2, %arg3], [%arg4, %arg5] : ui64 -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.store_nd %1, %2 : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @gemm_loop
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<1024x1024xbf16>, %[[ARG1:[0-9a-zA-Z]+]]: memref<1024x1024xbf16>, %[[ARG2:[0-9a-zA-Z]+]]: memref<1024x1024xf32>) {
-// CHECK: %[[BLOCK_ID_X:.*]] = gpu.block_id x
-// CHECK: %[[BLOCK_ID_Y:.*]] = gpu.block_id y
-// CHECK: %[[Y_COORD:.*]] = arith.muli %[[BLOCK_ID_Y]], %c16 : index
-// CHECK: %[[X_COORD:.*]] = arith.muli %[[BLOCK_ID_X]], %c8 : index
-// CHECK: %[[T2:.*]] = xegpu.create_nd_tdesc %[[ARG2]][%[[X_COORD]], %[[Y_COORD]]] : memref<1024x1024xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK: %[[T3:.*]] = xegpu.load_nd %[[T2]] : !xegpu.tensor_desc<8x16xf32> -> vector<8xf32>
-// CHECK: %[[T4:.*]] = vector.shape_cast %[[T3]] : vector<8xf32> to vector<8x1xf32>
-// CHECK: %[[T5:.*]] = scf.for %[[K:.*]] = %{{.*}} to %{{.*}} step %{{.*}} iter_args(%[[ARG4:.*]] = %[[T4]]) -> (vector<8x1xf32>) {
-// CHECK: %[[T10:.*]] = xegpu.create_nd_tdesc %[[ARG1]][%[[K]], %[[Y_COORD]]] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<16x16xbf16>
-// CHECK: %[[T11:.*]] = xegpu.load_nd %[[T10]] <{packed}> : !xegpu.tensor_desc<16x16xbf16> -> vector<16xbf16>
-// CHECK: %[[T12:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%[[X_COORD]], %[[K]]] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<8x16xbf16>
-// CHECK: %[[T13:.*]] = xegpu.load_nd %[[T12]] : !xegpu.tensor_desc<8x16xbf16> -> vector<8xbf16>
-// CHECK: %[[T14:.*]] = vector.shape_cast %[[ARG4]] : vector<8x1xf32> to vector<8xf32>
-// CHECK: %[[T15:.*]] = xegpu.dpas %[[T13]], %[[T11]], %[[T14]] : vector<8xbf16>, vector<16xbf16>, vector<8xf32> -> vector<8xf32>
-// CHECK: %[[T16:.*]] = vector.shape_cast %[[T15]] : vector<8xf32> to vector<8x1xf32>
-// CHECK: scf.yield %[[T16]] : vector<8x1xf32>
-// CHECK: }
-// CHECK: %[[T9:.*]] = vector.shape_cast %[[T5]] : vector<8x1xf32> to vector<8xf32>
-// CHECK: xegpu.store_nd %[[T9]], %[[T2]] : vector<8xf32>, !xegpu.tensor_desc<8x16xf32>
-gpu.module @test {
-gpu.func @gemm_loop(%arg0: memref<1024x1024xbf16>, %arg1: memref<1024x1024xbf16>, %arg2: memref<1024x1024xf32>){
-  %c0 = arith.constant 0 : index
-  %c16 = arith.constant 16 : index
-  %c8 = arith.constant 8 : index
-  %c1024 = arith.constant 1024 : index
-  %0 = gpu.block_id x
-  %1 = gpu.block_id y
-  %2 = arith.muli %0, %c8 : index
-  %3 = arith.muli %1, %c16 : index
-  %4 = xegpu.create_nd_tdesc %arg2[%2, %3] : memref<1024x1024xf32> -> !xegpu.tensor_desc<8x16xf32>
-  %5 = xegpu.load_nd %4 : !xegpu.tensor_desc<8x16xf32> -> vector<8x16xf32>
-  %6 = scf.for %arg3 = %c0 to %c1024 step %c16 iter_args(%arg4 = %5) -> (vector<8x16xf32>) {
-    %7 = xegpu.create_nd_tdesc %arg0[%2, %arg3] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<8x16xbf16>
-    %8 = xegpu.create_nd_tdesc %arg1[%arg3, %3] : memref<1024x1024xbf16> -> !xegpu.tensor_desc<16x16xbf16>
-    %9 = xegpu.load_nd %7 : !xegpu.tensor_desc<8x16xbf16> -> vector<8x16xbf16>
-    %10 = xegpu.load_nd %8 : !xegpu.tensor_desc<16x16xbf16> -> vector<16x16xbf16>
-    %11 = xegpu.dpas %9, %10, %arg4 : vector<8x16xbf16>, vector<16x16xbf16>, vector<8x16xf32> -> vector<8x16xf32>
-    scf.yield %11 : vector<8x16xf32>
-  }
-  xegpu.store_nd %6, %4 : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @update_nd_offset_1d(
-// CHECK: %[[ARG0:[0-9a-zA-Z]+]]: memref<256xf32>) {
-// CHECK: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<1xf32>
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
-// CHECK: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%c32] : !xegpu.tensor_desc<16xf32>
-// CHECK: xegpu.store_nd %[[CST]], %[[T1]]  : vector<1xf32>, !xegpu.tensor_desc<16xf32>
-gpu.module @test {
-gpu.func @update_nd_offset_1d(%arg0: memref<256xf32>){
-  %c0 = arith.constant 0 : index
-  %c32 = arith.constant 32 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
-  %2 = xegpu.update_nd_offset %0, [%c32] : !xegpu.tensor_desc<16xf32>
-  xegpu.store_nd %1, %2 : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @update_nd_offset_2d
-// CHECK: %[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf32>) {
-// CHECK: %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16xf32>
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
-// CHECK: %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%c32, %c32] : !xegpu.tensor_desc<16x16xf32>
-// CHECK: xegpu.store_nd %[[CST]], %[[T1]]  : vector<16xf32>, !xegpu.tensor_desc<16x16xf32>
-gpu.module @test {
-gpu.func @update_nd_offset_2d(%arg0: memref<256x256xf32>){
-  %c0 = arith.constant 0 : index
-  %c32 = arith.constant 32 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16x16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
-  %2 = xegpu.update_nd_offset %0, [%c32, %c32] : !xegpu.tensor_desc<16x16xf32>
-  xegpu.store_nd %1, %2 : vector<16x16xf32>, !xegpu.tensor_desc<16x16xf32>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @prefetch_2d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<256x256xf16>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16x16xf16>
-gpu.module @test {
-gpu.func @prefetch_2d(%arg0: memref<256x256xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16x16xf16>
-  gpu.return
-}
-}
-
-// -----
-// CHECK-LABEL: gpu.func @prefetch_1d
-// CHECK: (%[[ARG0:[0-9a-zA-Z]+]]: memref<256xf16>) {
-// CHECK: %[[T0:.*]] = xegpu.create_nd_tdesc %[[ARG0]][%{{.*}}] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
-// CHECK: xegpu.prefetch_nd %[[T0]] <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}> : !xegpu.tensor_desc<16xf16>
-gpu.module @test {
-gpu.func @prefetch_1d(%arg0: memref<256xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
-  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16xf16>
-  gpu.return
-}
-}
diff --git a/mlir/test/Dialect/XeGPU/subgroup-map-propagation.mlir b/mlir/test/Dialect/XeGPU/subgroup-map-propagation.mlir
deleted file mode 100644
index 35ac39d074c7..000000000000
--- a/mlir/test/Dialect/XeGPU/subgroup-map-propagation.mlir
+++ /dev/null
@@ -1,622 +0,0 @@
-// RUN: mlir-opt -xegpu-subgroup-distribute='print-analysis-only=true' -split-input-file %s | FileCheck %s
-
-// CHECK: function: dpas_f16:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %[[T1]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.dpas %[[T2]], %[[T3]], %{{.*}} : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %{{.*}} = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @dpas_f16(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %4 = xegpu.dpas %2, %3, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-
-// -----
-// CHECK: function: dpas_i8:
-// CHECK-NEXT: argument: <block argument> of type 'vector<8x32xi8>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 2]
-// CHECK-NEXT: argument: <block argument> of type 'vector<32x16xi8>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [4, 1]
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xi32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.dpas %{{.*}} : vector<8x32xi8>, vector<32x16xi8> -> vector<8x16xi32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xi32> -> !xegpu.tensor_desc<8x16xi32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @dpas_i8(%arg0: vector<8x32xi8>, %arg1: vector<32x16xi8>, %arg2: memref<8x16xi32>) {
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.dpas %arg0, %arg1 : vector<8x32xi8>, vector<32x16xi8> -> vector<8x16xi32>
-  %1 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xi32> -> !xegpu.tensor_desc<8x16xi32>
-  xegpu.store_nd %0, %1  : vector<8x16xi32>, !xegpu.tensor_desc<8x16xi32>
-  return
-}
-
-// -----
-// CHECK: function: load_with_transpose_effect:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [16, 1], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %[[T1]] <{transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.dpas %[[T2]], %[[T3]], %[[CST]] : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T5:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @load_with_transpose_effect(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %3 = xegpu.load_nd %1 <{transpose = array<i64: 1, 0>}> : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %4 = xegpu.dpas %2, %3, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: vector_transpose:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [16, 1], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %[[T1]]  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [16, 1], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T4:.*]] = vector.transpose %[[T3]], [1, 0] : vector<16x16xf16> to vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T5:.*]] = xegpu.dpas %[[T2]], %[[T4]], %[[CST]] : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T6:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @vector_transpose(%arg0: memref<8x16xf16>, %arg1: memref<16x16xf16>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %4 = vector.transpose %3, [1, 0] : vector<16x16xf16> to vector<16x16xf16>
-  %5 = xegpu.dpas %2, %4, %cst : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-  %6 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %5, %6  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: extf_truncf:
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = arith.extf %[[T1]] : vector<16x16xf16> to vector<16x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = arith.truncf %[[T2]] : vector<16x16xf32> to vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.dpas %[[T0]], %[[T3]] : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: Not assigned.
-func.func @extf_truncf(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>) -> vector<8x16xf32> {
-  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %2 = arith.extf %1 : vector<16x16xf16> to vector<16x16xf32>
-  %3 = arith.truncf %2 : vector<16x16xf32> to vector<16x16xf16>
-  %4 = xegpu.dpas %0, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  return %4 : vector<8x16xf32>
-}
-
-// -----
-// CHECK: function: load_gather_with_transpose_effect:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<256xf16>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST0:.*]] = arith.constant dense<true> : vector<16xi1>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.create_tdesc %{{.*}}, %[[CST]] : memref<256xf16>, vector<16xindex> -> !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>
-// CHECK-NEXT: layout for result #0: lane_layout: [16, 1], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load %[[T2]], %[[CST0]] <{transpose}> : !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>, vector<16xi1> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.dpas %[[T1]], %[[T3]] : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T5:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @load_gather_with_transpose_effect(%arg0: memref<8x16xf16>, %arg1: memref<256xf16>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-  %cst_0 = arith.constant dense<true> : vector<16xi1>
-  %2 = xegpu.create_tdesc %arg1, %cst : memref<256xf16>, vector<16xindex> -> !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>
-  %3 = xegpu.load %2, %cst_0 <{transpose}> : !xegpu.tensor_desc<16x16xf16, #xegpu.scatter_tdesc_attr<chunk_size = 16 : i64>>, vector<16xi1> -> vector<16x16xf16>
-  %4 = xegpu.dpas %1, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  %5 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %4, %5  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: load_gather_1d:
-// CHECK: argument: <block argument> of type 'memref<256xf32>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16xf32>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST0:.*]] = arith.constant dense<true> : vector<16xi1>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_tdesc %{{.*}}, %[[CST]] : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T1]] = xegpu.load %[[T0]], %[[CST0]]  : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @load_gather_1d(%arg0: memref<256xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
-  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-  %cst_0 = arith.constant dense<true> : vector<16xi1>
-  %0 = xegpu.create_tdesc %arg0, %cst : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-  %1 = xegpu.load %0, %cst_0  : !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1> -> vector<16xf32>
-  xegpu.store_nd %1, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  return
-}
-
-// -----
-// CHECK: function: store_scatter_with_transpose_effect:
-// CHECK-NEXT: argument: <block argument> of type 'memref<128xf32>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[CST0:.*]] = arith.constant dense<true> : vector<16xi1>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST1:.*]] = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_tdesc %{{.*}}, %[[CST1]] : memref<128xf32>, vector<16xindex> -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
-// CHECK-NEXT: layout for result #0: lane_layout: [16, 1], lane_data: [1, 1]
-func.func @store_scatter_with_transpose_effect(%arg0: memref<128xf32>) {
-  %cst = arith.constant dense<1.000000e+00> : vector<8x16xf32>
-  %cst_0 = arith.constant dense<true> : vector<16xi1>
-  %cst_1 = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-  %0 = xegpu.create_tdesc %arg0, %cst_1 : memref<128xf32>, vector<16xindex> -> !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>
-  xegpu.store %cst, %0, %cst_0 <{transpose}> : vector<8x16xf32>, !xegpu.tensor_desc<16x8xf32, #xegpu.scatter_tdesc_attr<chunk_size = 8 : i64>>, vector<16xi1>
-  return
-}
-
-// -----
-// CHECK: function: store_scatter_1d:
-// CHECK-NEXT: argument: <block argument> of type 'vector<16xf32>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: argument: <block argument> of type 'memref<256xf32>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST1:.*]] = arith.constant dense<true> : vector<16xi1>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_tdesc %{{.*}}, %[[CST]] : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @store_scatter_1d(%arg0: vector<16xf32>, %arg1: memref<256xf32>) {
-  %cst = arith.constant dense<[0, 16, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240]> : vector<16xindex>
-  %cst_0 = arith.constant dense<true> : vector<16xi1>
-  %0 = xegpu.create_tdesc %arg1, %cst : memref<256xf32>, vector<16xindex> -> !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>
-  xegpu.store %arg0, %0, %cst_0  : vector<16xf32>, !xegpu.tensor_desc<16xf32, #xegpu.scatter_tdesc_attr<>>, vector<16xi1>
-  return
-}
-
-// -----
-// CHECK: function: vector_bitcast_i16_to_i8:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xi16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<32x16xi8>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xi32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xi16> -> !xegpu.tensor_desc<8x16xi16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<32x16xi8> -> !xegpu.tensor_desc<32x16xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [4, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x16xi16> -> vector<8x16xi16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %[[T1]]  : !xegpu.tensor_desc<32x16xi8> -> vector<32x16xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [4, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = vector.bitcast %[[T2]] : vector<8x16xi16> to vector<8x32xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T5:.*]] = xegpu.dpas %[[T4]], %[[T3]] : vector<8x32xi8>, vector<32x16xi8> -> vector<8x16xi32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T6:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xi32> -> !xegpu.tensor_desc<8x16xi32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @vector_bitcast_i16_to_i8(%arg0: memref<8x16xi16>, %arg1: memref<32x16xi8>, %arg2: memref<8x16xi32>) {
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x16xi16> -> !xegpu.tensor_desc<8x16xi16>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<32x16xi8> -> !xegpu.tensor_desc<32x16xi8>
-  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x16xi16> -> vector<8x16xi16>
-  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<32x16xi8> -> vector<32x16xi8>
-  %4 = vector.bitcast %2 : vector<8x16xi16> to vector<8x32xi8>
-  %5 = xegpu.dpas %4, %3 : vector<8x32xi8>, vector<32x16xi8> -> vector<8x16xi32>
-  %6 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xi32> -> !xegpu.tensor_desc<8x16xi32>
-  xegpu.store_nd %5, %6  : vector<8x16xi32>, !xegpu.tensor_desc<8x16xi32>
-  return
-}
-
-// -----
-// CHECK: function: vector_bitcast_i8_to_f16:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x32xi8>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<16x32xi8>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x32xi8> -> !xegpu.tensor_desc<8x32xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<16x32xi8> -> !xegpu.tensor_desc<16x32xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [4, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %[[T0]]  : !xegpu.tensor_desc<8x32xi8> -> vector<8x32xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 2]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %[[T1]]  : !xegpu.tensor_desc<16x32xi8> -> vector<16x32xi8>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [4, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = vector.bitcast %[[T2]] : vector<8x32xi8> to vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T5:.*]] = vector.bitcast %[[T3]] : vector<16x32xi8> to vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T6:.*]] = xegpu.dpas %[[T4]], %[[T5]] : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T7:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @vector_bitcast_i8_to_f16(%arg0: memref<8x32xi8>, %arg1: memref<16x32xi8>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x32xi8> -> !xegpu.tensor_desc<8x32xi8>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<16x32xi8> -> !xegpu.tensor_desc<16x32xi8>
-  %2 = xegpu.load_nd %0  : !xegpu.tensor_desc<8x32xi8> -> vector<8x32xi8>
-  %3 = xegpu.load_nd %1  : !xegpu.tensor_desc<16x32xi8> -> vector<16x32xi8>
-  %4 = vector.bitcast %2 : vector<8x32xi8> to vector<8x16xf16>
-  %5 = vector.bitcast %3 : vector<16x32xi8> to vector<16x16xf16>
-  %6 = xegpu.dpas %4, %5 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  %7 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %6, %7  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: binary_op_one_use:
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = arith.addf %[[T1]], %[[T2]] : vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.dpas %[[T0]], %[[T3]] : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @binary_op_one_use(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: !xegpu.tensor_desc<8x16xf32>) {
-  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %2 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %3 = arith.addf %1, %2 : vector<16x16xf16>
-  %4 = xegpu.dpas %0, %3 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  xegpu.store_nd %4, %arg2  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: binary_op_multiple_uses:
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 3
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = arith.addf %[[T1]], %[[CST]] : vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.dpas %[[T0]], %[[T2]] : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @binary_op_multiple_uses(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: !xegpu.tensor_desc<8x16xf32>, %arg3: !xegpu.tensor_desc<16x16xf16>) {
-  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %1 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-  %cst = arith.constant dense<1.000000e+00> : vector<16x16xf16>
-  %2 = arith.addf %1, %cst : vector<16x16xf16>
-  %3 = xegpu.dpas %0, %2 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  xegpu.store_nd %3, %arg2  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %2, %arg3  : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  return
-}
-
-// -----
-// CHECK: function: for_op:
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x128xf16>' at index: 0
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<128x16xf16>' at index: 1
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type 'memref<8x16xf32>' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 0 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 128 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %{{.*}} = arith.constant 16 : index
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x128xf16> -> !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<128x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T5:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T6:.*]] = xegpu.dpas %[[T4]], %[[T5]], %{{.*}} : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T7:.*]] = xegpu.update_nd_offset %{{.*}} : !xegpu.tensor_desc<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T8:.*]] = xegpu.update_nd_offset %{{.*}} : !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : scf.for
-// CHECK-NEXT: layout for result #0: Not assigned.
-// CHECK-NEXT: layout for result #1: Not assigned.
-// CHECK-NEXT: layout for result #2: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.create_nd_tdesc %{{.*}} : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @for_op(%arg0: memref<8x128xf16>, %arg1: memref<128x16xf16>, %arg2: memref<8x16xf32>) {
-  %c0 = arith.constant 0 : index
-  %c128 = arith.constant 128 : index
-  %c16 = arith.constant 16 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<8x128xf16> -> !xegpu.tensor_desc<8x16xf16>
-  %1 = xegpu.create_nd_tdesc %arg1[%c0, %c0] : memref<128x16xf16> -> !xegpu.tensor_desc<16x16xf16>
-  %cst = arith.constant dense<0.000000e+00> : vector<8x16xf32>
-  %2:3 = scf.for %arg3 = %c0 to %c128 step %c16 iter_args(%arg4 = %0, %arg5 = %1, %arg6 = %cst) -> (!xegpu.tensor_desc<8x16xf16>, !xegpu.tensor_desc<16x16xf16>, vector<8x16xf32>) {
-    %4 = xegpu.load_nd %arg4  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-    %5 = xegpu.load_nd %arg5  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-    %6 = xegpu.dpas %4, %5, %arg6 : vector<8x16xf16>, vector<16x16xf16>, vector<8x16xf32> -> vector<8x16xf32>
-    %7 = xegpu.update_nd_offset %arg4, [%c0, %c16] : !xegpu.tensor_desc<8x16xf16>
-    %8 = xegpu.update_nd_offset %arg5, [%c16, %c0] : !xegpu.tensor_desc<16x16xf16>
-    scf.yield %7, %8, %6 : !xegpu.tensor_desc<8x16xf16>, !xegpu.tensor_desc<16x16xf16>, vector<8x16xf32>
-  }
-  %3 = xegpu.create_nd_tdesc %arg2[%c0, %c0] : memref<8x16xf32> -> !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %2#2, %3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: if_single_use:
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: argument: <block argument> of type 'i1' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf32>' at index: 3
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : scf.if
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [2, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.dpas %[[T0]], %{{.*}} : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @if_single_use(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: i1, %arg3: !xegpu.tensor_desc<8x16xf32>) {
-  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %1 = scf.if %arg2 -> (vector<16x16xf16>) {
-    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-    scf.yield %3 : vector<16x16xf16>
-  } else {
-    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-    scf.yield %3 : vector<16x16xf16>
-  }
-  %2 = xegpu.dpas %0, %1 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  xegpu.store_nd %2, %arg3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: if_multiple_uses:
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf16>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type 'i1' at index: 2
-// CHECK-NEXT: layout  : Not assigned.
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<8x16xf32>' at index: 3
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16x16xf16>' at index: 4
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T3:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T4:.*]] = xegpu.load_nd %{{.*}}  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : scf.if
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T2:.*]] = xegpu.dpas %[[T0]], %{{.*}} : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @if_multiple_uses(%arg0: !xegpu.tensor_desc<8x16xf16>, %arg1: !xegpu.tensor_desc<16x16xf16>, %arg2: i1, %arg3: !xegpu.tensor_desc<8x16xf32>, %arg4: !xegpu.tensor_desc<16x16xf16>) {
-  %0 = xegpu.load_nd %arg0  : !xegpu.tensor_desc<8x16xf16> -> vector<8x16xf16>
-  %1 = scf.if %arg2 -> (vector<16x16xf16>) {
-    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-    scf.yield %3 : vector<16x16xf16>
-  } else {
-    %3 = xegpu.load_nd %arg1  : !xegpu.tensor_desc<16x16xf16> -> vector<16x16xf16>
-    scf.yield %3 : vector<16x16xf16>
-  }
-  %2 = xegpu.dpas %0, %1 : vector<8x16xf16>, vector<16x16xf16> -> vector<8x16xf32>
-  xegpu.store_nd %2, %arg3  : vector<8x16xf32>, !xegpu.tensor_desc<8x16xf32>
-  xegpu.store_nd %1, %arg4  : vector<16x16xf16>, !xegpu.tensor_desc<16x16xf16>
-  return
-}
-
-// -----
-// CHECK: function: vector_outer_reduction:
-// CHECK-NEXT: argument: <block argument> of type 'vector<16x16xf32>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16xf32>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = vector.multi_reduction <add>, %{{.*}}, %[[CST]] [0] : vector<16x16xf32> to vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @vector_outer_reduction(%arg0: vector<16x16xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
-  %cst = arith.constant dense<0.000000e+00> : vector<16xf32>
-  %0 = vector.multi_reduction <add>, %arg0, %cst [0] : vector<16x16xf32> to vector<16xf32>
-  xegpu.store_nd %0, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  return
-}
-
-// -----
-// CHECK: function: vector_inner_reduction:
-// CHECK-NEXT: argument: <block argument> of type 'vector<16x16xf32>' at index: 0
-// CHECK-NEXT: layout  : lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: argument: <block argument> of type '!xegpu.tensor_desc<16xf32>' at index: 1
-// CHECK-NEXT: layout  : lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = vector.multi_reduction <add>, %{{.*}}, %[[CST]] [1] : vector<16x16xf32> to vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @vector_inner_reduction(%arg0: vector<16x16xf32>, %arg1: !xegpu.tensor_desc<16xf32>) {
-  %cst = arith.constant dense<0.000000e+00> : vector<16xf32>
-  %0 = vector.multi_reduction <add>, %arg0, %cst [1] : vector<16x16xf32> to vector<16xf32>
-  xegpu.store_nd %0, %arg1  : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  return
-}
-
-// -----
-// CHECK: function: update_nd_offset_1d:
-// CHECK: op    : %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}}[%{{.*}}] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%{{.*}}] : !xegpu.tensor_desc<16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @update_nd_offset_1d(%arg0: memref<256xf32>){
-  %c0 = arith.constant 0 : index
-  %c32 = arith.constant 32 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf32> -> !xegpu.tensor_desc<16xf32>
-  %2 = xegpu.update_nd_offset %0, [%c32] : !xegpu.tensor_desc<16xf32>
-  xegpu.store_nd %1, %2 : vector<16xf32>, !xegpu.tensor_desc<16xf32>
-  return
-}
-
-// -----
-// CHECK: function: update_nd_offset_2d:
-// CHECK: op    : %[[CST:.*]] = arith.constant dense<1.000000e+00> : vector<16x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}}[%{{.*}}] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-// CHECK-NEXT: op    : %[[T1:.*]] = xegpu.update_nd_offset %[[T0]], [%{{.*}}] : !xegpu.tensor_desc<16x16xf32>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @update_nd_offset_2d(%arg0: memref<256x256xf32>){
-  %c0 = arith.constant 0 : index
-  %c32 = arith.constant 32 : index
-  %1 = arith.constant dense<1.000000e+00> : vector<16x16xf32>
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf32> -> !xegpu.tensor_desc<16x16xf32>
-  %2 = xegpu.update_nd_offset %0, [%c32, %c32] : !xegpu.tensor_desc<16x16xf32>
-  xegpu.store_nd %1, %2 : vector<16x16xf32>, !xegpu.tensor_desc<16x16xf32>
-  return
-}
-
-// -----
-// CHECK: function: prefetch_2d:
-// CHECK: layout for result #0: Not assigned.
-// CHECK-NEXT: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}}[%{{.*}}] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [1, 16], lane_data: [1, 1]
-func.func @prefetch_2d(%arg0: memref<256x256xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0, %c0] : memref<256x256xf16> -> !xegpu.tensor_desc<16x16xf16>
-  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16x16xf16>
-  return
-}
-
-// -----
-// CHECK: function: prefetch_1d:
-// CHECK: op    : %[[T0:.*]] = xegpu.create_nd_tdesc %{{.*}}[%{{.*}}] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
-// CHECK-NEXT: layout for result #0: lane_layout: [16], lane_data: [1]
-func.func @prefetch_1d(%arg0: memref<256xf16>){
-  %c0 = arith.constant 0 : index
-  %0 = xegpu.create_nd_tdesc %arg0[%c0] : memref<256xf16> -> !xegpu.tensor_desc<16xf16>
-  xegpu.prefetch_nd %0 <{l1_hint = #xegpu.cache_hint<cached>, l2_hint = #xegpu.cache_hint<uncached>}>: !xegpu.tensor_desc<16xf16>
-  return
-}
-- 
cgit v1.2.3


From 04e2e581ac000934782398e05853338040bf7c46 Mon Sep 17 00:00:00 2001
From: Craig Topper <craig.topper@sifive.com>
Date: Fri, 20 Jun 2025 10:44:51 -0700
Subject: [RISCV] Treat bf16->f32 as separate ExtKind in combineOp_VLToVWOp_VL.
 (#144653)

This allows us to better track the narrow type we need and to fix
miscompiles if f16->f32 and bf16->f32 extends are mixed.

Fixes #144651.
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp        | 78 ++++++++++++++--------
 .../CodeGen/RISCV/rvv/fixed-vectors-vfwmaccbf16.ll | 58 ++++++++++++++--
 2 files changed, 103 insertions(+), 33 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index b8ef221742a2..139fa7ba3562 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -16316,7 +16316,12 @@ namespace {
 // apply a combine.
 struct CombineResult;
 
-enum ExtKind : uint8_t { ZExt = 1 << 0, SExt = 1 << 1, FPExt = 1 << 2 };
+enum ExtKind : uint8_t {
+  ZExt = 1 << 0,
+  SExt = 1 << 1,
+  FPExt = 1 << 2,
+  BF16Ext = 1 << 3
+};
 /// Helper class for folding sign/zero extensions.
 /// In particular, this class is used for the following combines:
 /// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
@@ -16351,8 +16356,10 @@ struct NodeExtensionHelper {
   /// instance, a splat constant (e.g., 3), would support being both sign and
   /// zero extended.
   bool SupportsSExt;
-  /// Records if this operand is like being floating-Point extended.
+  /// Records if this operand is like being floating point extended.
   bool SupportsFPExt;
+  /// Records if this operand is extended from bf16.
+  bool SupportsBF16Ext;
   /// This boolean captures whether we care if this operand would still be
   /// around after the folding happens.
   bool EnforceOneUse;
@@ -16388,6 +16395,7 @@ struct NodeExtensionHelper {
     case ExtKind::ZExt:
       return RISCVISD::VZEXT_VL;
     case ExtKind::FPExt:
+    case ExtKind::BF16Ext:
       return RISCVISD::FP_EXTEND_VL;
     }
     llvm_unreachable("Unknown ExtKind enum");
@@ -16409,13 +16417,6 @@ struct NodeExtensionHelper {
     if (Source.getValueType() == NarrowVT)
       return Source;
 
-    // vfmadd_vl -> vfwmadd_vl can take bf16 operands
-    if (Source.getValueType().getVectorElementType() == MVT::bf16) {
-      assert(Root->getSimpleValueType(0).getVectorElementType() == MVT::f32 &&
-             Root->getOpcode() == RISCVISD::VFMADD_VL);
-      return Source;
-    }
-
     unsigned ExtOpc = getExtOpc(*SupportsExt);
 
     // If we need an extension, we should be changing the type.
@@ -16458,7 +16459,8 @@ struct NodeExtensionHelper {
     // Determine the narrow size.
     unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
 
-    MVT EltVT = SupportsExt == ExtKind::FPExt
+    MVT EltVT = SupportsExt == ExtKind::BF16Ext ? MVT::bf16
+                : SupportsExt == ExtKind::FPExt
                     ? MVT::getFloatingPointVT(NarrowSize)
                     : MVT::getIntegerVT(NarrowSize);
 
@@ -16635,17 +16637,13 @@ struct NodeExtensionHelper {
     EnforceOneUse = false;
   }
 
-  bool isSupportedFPExtend(SDNode *Root, MVT NarrowEltVT,
-                           const RISCVSubtarget &Subtarget) {
-    // Any f16 extension will need zvfh
-    if (NarrowEltVT == MVT::f16 && !Subtarget.hasVInstructionsF16())
-      return false;
-    // The only bf16 extension we can do is vfmadd_vl -> vfwmadd_vl with
-    // zvfbfwma
-    if (NarrowEltVT == MVT::bf16 && (!Subtarget.hasStdExtZvfbfwma() ||
-                                     Root->getOpcode() != RISCVISD::VFMADD_VL))
-      return false;
-    return true;
+  bool isSupportedFPExtend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
+    return (NarrowEltVT == MVT::f32 ||
+            (NarrowEltVT == MVT::f16 && Subtarget.hasVInstructionsF16()));
+  }
+
+  bool isSupportedBF16Extend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
+    return NarrowEltVT == MVT::bf16 && Subtarget.hasStdExtZvfbfwma();
   }
 
   /// Helper method to set the various fields of this struct based on the
@@ -16655,6 +16653,7 @@ struct NodeExtensionHelper {
     SupportsZExt = false;
     SupportsSExt = false;
     SupportsFPExt = false;
+    SupportsBF16Ext = false;
     EnforceOneUse = true;
     unsigned Opc = OrigOperand.getOpcode();
     // For the nodes we handle below, we end up using their inputs directly: see
@@ -16686,9 +16685,11 @@ struct NodeExtensionHelper {
     case RISCVISD::FP_EXTEND_VL: {
       MVT NarrowEltVT =
           OrigOperand.getOperand(0).getSimpleValueType().getVectorElementType();
-      if (!isSupportedFPExtend(Root, NarrowEltVT, Subtarget))
-        break;
-      SupportsFPExt = true;
+      if (isSupportedFPExtend(NarrowEltVT, Subtarget))
+        SupportsFPExt = true;
+      if (isSupportedBF16Extend(NarrowEltVT, Subtarget))
+        SupportsBF16Ext = true;
+
       break;
     }
     case ISD::SPLAT_VECTOR:
@@ -16705,16 +16706,16 @@ struct NodeExtensionHelper {
       if (Op.getOpcode() != ISD::FP_EXTEND)
         break;
 
-      if (!isSupportedFPExtend(Root, Op.getOperand(0).getSimpleValueType(),
-                               Subtarget))
-        break;
-
       unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
       unsigned ScalarBits = Op.getOperand(0).getValueSizeInBits();
       if (NarrowSize != ScalarBits)
         break;
 
-      SupportsFPExt = true;
+      if (isSupportedFPExtend(Op.getOperand(0).getSimpleValueType(), Subtarget))
+        SupportsFPExt = true;
+      if (isSupportedBF16Extend(Op.getOperand(0).getSimpleValueType(),
+                                Subtarget))
+        SupportsBF16Ext = true;
       break;
     }
     default:
@@ -16947,6 +16948,11 @@ canFoldToVWWithSameExtensionImpl(SDNode *Root, const NodeExtensionHelper &LHS,
     return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
                          Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
                          /*RHSExt=*/{ExtKind::FPExt});
+  if ((AllowExtMask & ExtKind::BF16Ext) && LHS.SupportsBF16Ext &&
+      RHS.SupportsBF16Ext)
+    return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
+                         Root, LHS, /*LHSExt=*/{ExtKind::BF16Ext}, RHS,
+                         /*RHSExt=*/{ExtKind::BF16Ext});
   return std::nullopt;
 }
 
@@ -17029,6 +17035,18 @@ canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS,
                                           Subtarget);
 }
 
+/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS))
+///
+/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
+/// can be used to apply the pattern.
+static std::optional<CombineResult>
+canFoldToVWWithBF16EXT(SDNode *Root, const NodeExtensionHelper &LHS,
+                       const NodeExtensionHelper &RHS, SelectionDAG &DAG,
+                       const RISCVSubtarget &Subtarget) {
+  return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::BF16Ext, DAG,
+                                          Subtarget);
+}
+
 /// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS))
 ///
 /// \returns std::nullopt if the pattern doesn't match or a CombineResult that
@@ -17068,6 +17086,8 @@ NodeExtensionHelper::getSupportedFoldings(const SDNode *Root) {
   case RISCVISD::VFNMADD_VL:
   case RISCVISD::VFNMSUB_VL:
     Strategies.push_back(canFoldToVWWithSameExtension);
+    if (Root->getOpcode() == RISCVISD::VFMADD_VL)
+      Strategies.push_back(canFoldToVWWithBF16EXT);
     break;
   case ISD::MUL:
   case RISCVISD::MUL_VL:
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmaccbf16.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmaccbf16.ll
index 1639f21f243d..aec970adff51 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmaccbf16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vfwmaccbf16.ll
@@ -1,8 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfbfwma -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFWMA
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfbfwma -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFWMA
-; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfbfmin -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFMIN
-; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfbfmin -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFMIN
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfwma -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFWMA
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfwma -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFWMA
+; RUN: llc < %s -mtriple=riscv32 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFMIN
+; RUN: llc < %s -mtriple=riscv64 -mattr=+v,+zvfh,+zvfbfmin -verify-machineinstrs | FileCheck %s --check-prefix=ZVFBFMIN
 
 define <1 x float> @vfwmaccbf16_vv_v1f32(<1 x float> %a, <1 x bfloat> %b, <1 x bfloat> %c) {
 ; ZVFBFWMA-LABEL: vfwmaccbf16_vv_v1f32:
@@ -295,3 +295,53 @@ define <32 x float> @vfwmaccbf32_vf_v32f32(<32 x float> %a, bfloat %b, <32 x bfl
   %res = call <32 x float> @llvm.fma.v32f32(<32 x float> %b.ext, <32 x float> %c.ext, <32 x float> %a)
   ret <32 x float> %res
 }
+
+define <4 x float> @vfwmaccbf16_vf_v4f32_scalar_extend(<4 x float> %rd, bfloat %a, <4 x bfloat> %b) local_unnamed_addr #0 {
+; ZVFBFWMA-LABEL: vfwmaccbf16_vf_v4f32_scalar_extend:
+; ZVFBFWMA:       # %bb.0:
+; ZVFBFWMA-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFBFWMA-NEXT:    vfwmaccbf16.vf v8, fa0, v9
+; ZVFBFWMA-NEXT:    ret
+;
+; ZVFBFMIN-LABEL: vfwmaccbf16_vf_v4f32_scalar_extend:
+; ZVFBFMIN:       # %bb.0:
+; ZVFBFMIN-NEXT:    fmv.x.w a0, fa0
+; ZVFBFMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFBFMIN-NEXT:    vfwcvtbf16.f.f.v v10, v9
+; ZVFBFMIN-NEXT:    slli a0, a0, 16
+; ZVFBFMIN-NEXT:    fmv.w.x fa5, a0
+; ZVFBFMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFBFMIN-NEXT:    vfmacc.vf v8, fa5, v10
+; ZVFBFMIN-NEXT:    ret
+  %b_ext = fpext <4 x bfloat> %b to <4 x float>
+  %a_extend = fpext bfloat %a to float
+  %a_insert = insertelement <4 x float> poison, float %a_extend, i64 0
+  %a_shuffle = shufflevector <4 x float> %a_insert, <4 x float> poison, <4 x i32> zeroinitializer
+  %fma = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %a_shuffle, <4 x float> %b_ext, <4 x float> %rd)
+  ret <4 x float> %fma
+}
+
+; Negative test with a mix of bfloat and half fpext.
+define <4 x float> @mix(<4 x float> %rd, <4 x half> %a, <4 x bfloat> %b) {
+; ZVFBFWMA-LABEL: mix:
+; ZVFBFWMA:       # %bb.0:
+; ZVFBFWMA-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFBFWMA-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFBFWMA-NEXT:    vfwcvtbf16.f.f.v v9, v10
+; ZVFBFWMA-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFBFWMA-NEXT:    vfmacc.vv v8, v11, v9
+; ZVFBFWMA-NEXT:    ret
+;
+; ZVFBFMIN-LABEL: mix:
+; ZVFBFMIN:       # %bb.0:
+; ZVFBFMIN-NEXT:    vsetivli zero, 4, e16, mf2, ta, ma
+; ZVFBFMIN-NEXT:    vfwcvt.f.f.v v11, v9
+; ZVFBFMIN-NEXT:    vfwcvtbf16.f.f.v v9, v10
+; ZVFBFMIN-NEXT:    vsetvli zero, zero, e32, m1, ta, ma
+; ZVFBFMIN-NEXT:    vfmacc.vv v8, v11, v9
+; ZVFBFMIN-NEXT:    ret
+  %a_ext = fpext <4 x half> %a to <4 x float>
+  %b_ext = fpext <4 x bfloat> %b to <4 x float>
+  %fma = tail call <4 x float> @llvm.fma.v4f32(<4 x float> %a_ext, <4 x float> %b_ext, <4 x float> %rd)
+  ret <4 x float> %fma
+}
-- 
cgit v1.2.3


From ab8b8c1e138ae705f251626b63ad2cf4d7937003 Mon Sep 17 00:00:00 2001
From: Sam Elliott <quic_aelliott@quicinc.com>
Date: Fri, 20 Jun 2025 11:05:25 -0700
Subject: [TargetParser][cmake] Be Smarter about TableGen Deps (#144848)

This tries to be a bit smarter for the OLD behaviour of CMP0116, to glob
more relevant directories looking for possible dependencies.

The changes are:
- Remove some duplication of lines in the `tablegen` function.
- Put CURRENT_SOURCE_DIR into `tblgen_includes` (at the front)
- Glob all directories in `tblgen_includes`
- Give up on `local_tds` which was wrong when using tablegen to compile
a file in a different directory (as TargetParser does)
- Use `EXTRA_INCLUDES` in TargetParser `tablegen` calls.

This is still an under-approximation of what might be included, at least
comparing the RISCVTargetParserDef.inc.d (after building
`target_parser_gen`), and the list of deps in the ninja file when
explicitly setting CMP0116 to OLD.

Fixes #144639
---
 llvm/cmake/modules/TableGen.cmake             | 59 ++++++++++++---------------
 llvm/include/llvm/TargetParser/CMakeLists.txt |  6 +--
 2 files changed, 28 insertions(+), 37 deletions(-)

diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index b26fc62d4cc0..67a628d4953c 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -21,6 +21,13 @@ function(tablegen project ofn)
     message(FATAL_ERROR "${project}_TABLEGEN_EXE not set")
   endif()
 
+  # Set the include directories
+  get_directory_property(tblgen_includes INCLUDE_DIRECTORIES)
+  list(PREPEND tblgen_includes ${ARG_EXTRA_INCLUDES})
+  list(PREPEND tblgen_includes ${CMAKE_CURRENT_SOURCE_DIR})
+  # Filter out any empty include items.
+  list(REMOVE_ITEM tblgen_includes "")
+
   # Use depfile instead of globbing arbitrary *.td(s) for Ninja. We force
   # CMake versions older than v3.30 on Windows to use the fallback behavior
   # due to a depfile parsing bug on Windows paths in versions prior to 3.30.
@@ -42,22 +49,16 @@ function(tablegen project ofn)
       -d ${ofn}.d
       DEPFILE ${ofn}.d
       )
-    set(local_tds)
     set(global_tds)
   else()
-    file(GLOB local_tds "*.td")
-    file(GLOB_RECURSE global_tds "${LLVM_MAIN_INCLUDE_DIR}/llvm/*.td")
+    set(include_td_dirs "${tblgen_includes}")
+    list(TRANSFORM include_td_dirs APPEND "/*.td")
+    file(GLOB global_tds ${include_td_dirs})
     set(additional_cmdline
       -o ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
       )
   endif()
 
-  if (IS_ABSOLUTE ${LLVM_TARGET_DEFINITIONS})
-    set(LLVM_TARGET_DEFINITIONS_ABSOLUTE ${LLVM_TARGET_DEFINITIONS})
-  else()
-    set(LLVM_TARGET_DEFINITIONS_ABSOLUTE
-      ${CMAKE_CURRENT_SOURCE_DIR}/${LLVM_TARGET_DEFINITIONS})
-  endif()
   if (LLVM_ENABLE_DAGISEL_COV AND "-gen-dag-isel" IN_LIST ARGN)
     list(APPEND LLVM_TABLEGEN_FLAGS "-instrument-coverage")
   endif()
@@ -92,30 +93,12 @@ function(tablegen project ofn)
     list(APPEND LLVM_TABLEGEN_FLAGS "-no-warn-on-unused-template-args")
   endif()
 
-  # We need both _TABLEGEN_TARGET and _TABLEGEN_EXE in the  DEPENDS list
-  # (both the target and the file) to have .inc files rebuilt on
-  # a tablegen change, as cmake does not propagate file-level dependencies
-  # of custom targets. See the following ticket for more information:
-  # https://cmake.org/Bug/view.php?id=15858
-  # The dependency on both, the target and the file, produces the same
-  # dependency twice in the result file when
-  # ("${${project}_TABLEGEN_TARGET}" STREQUAL "${${project}_TABLEGEN_EXE}")
-  # but lets us having smaller and cleaner code here.
-  get_directory_property(tblgen_includes INCLUDE_DIRECTORIES)
-  list(APPEND tblgen_includes ${ARG_EXTRA_INCLUDES})
-
-  # Get the current set of include paths for this td file.
-  cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES" ${ARGN})
-  get_directory_property(tblgen_includes INCLUDE_DIRECTORIES)
-  list(APPEND tblgen_includes ${ARG_EXTRA_INCLUDES})
-  # Filter out any empty include items.
-  list(REMOVE_ITEM tblgen_includes "")
-
   # Build the absolute path for the current input file.
   if (IS_ABSOLUTE ${LLVM_TARGET_DEFINITIONS})
     set(LLVM_TARGET_DEFINITIONS_ABSOLUTE ${LLVM_TARGET_DEFINITIONS})
   else()
-    set(LLVM_TARGET_DEFINITIONS_ABSOLUTE ${CMAKE_CURRENT_SOURCE_DIR}/${LLVM_TARGET_DEFINITIONS})
+    set(LLVM_TARGET_DEFINITIONS_ABSOLUTE
+      ${CMAKE_CURRENT_SOURCE_DIR}/${LLVM_TARGET_DEFINITIONS})
   endif()
 
   # Append this file and its includes to the compile commands file.
@@ -123,13 +106,21 @@ function(tablegen project ofn)
   file(APPEND ${CMAKE_BINARY_DIR}/tablegen_compile_commands.yml
       "--- !FileInfo:\n"
       "  filepath: \"${LLVM_TARGET_DEFINITIONS_ABSOLUTE}\"\n"
-      "  includes: \"${CMAKE_CURRENT_SOURCE_DIR};${tblgen_includes}\"\n"
+      "  includes: \"${tblgen_includes}\"\n"
   )
 
-  # Filter out empty items before prepending each entry with -I
-  list(REMOVE_ITEM tblgen_includes "")
+  # Prepend each include entry with -I for arguments.
   list(TRANSFORM tblgen_includes PREPEND -I)
 
+  # We need both _TABLEGEN_TARGET and _TABLEGEN_EXE in the  DEPENDS list
+  # (both the target and the file) to have .inc files rebuilt on
+  # a tablegen change, as cmake does not propagate file-level dependencies
+  # of custom targets. See the following ticket for more information:
+  # https://cmake.org/Bug/view.php?id=15858
+  # The dependency on both, the target and the file, produces the same
+  # dependency twice in the result file when
+  # ("${${project}_TABLEGEN_TARGET}" STREQUAL "${${project}_TABLEGEN_EXE}")
+  # but lets us having smaller and cleaner code here.
   set(tablegen_exe ${${project}_TABLEGEN_EXE})
   set(tablegen_depends ${${project}_TABLEGEN_TARGET} ${tablegen_exe})
 
@@ -140,7 +131,7 @@ function(tablegen project ofn)
   endif()
 
   add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${ofn}
-    COMMAND ${tablegen_exe} ${ARG_UNPARSED_ARGUMENTS} -I ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND ${tablegen_exe} ${ARG_UNPARSED_ARGUMENTS}
     ${tblgen_includes}
     ${LLVM_TABLEGEN_FLAGS}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
@@ -150,7 +141,7 @@ function(tablegen project ofn)
     # directory and local_tds may not contain it, so we must
     # explicitly list it here:
     DEPENDS ${ARG_DEPENDS} ${tablegen_depends}
-      ${local_tds} ${global_tds}
+      ${global_tds}
     ${LLVM_TARGET_DEFINITIONS_ABSOLUTE}
     ${LLVM_TARGET_DEPENDS}
     ${LLVM_TABLEGEN_JOB_POOL}
diff --git a/llvm/include/llvm/TargetParser/CMakeLists.txt b/llvm/include/llvm/TargetParser/CMakeLists.txt
index bb6d58d74a35..3811367be614 100644
--- a/llvm/include/llvm/TargetParser/CMakeLists.txt
+++ b/llvm/include/llvm/TargetParser/CMakeLists.txt
@@ -1,11 +1,11 @@
 set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/ARM/ARM.td)
-tablegen(LLVM ARMTargetParserDef.inc -gen-arm-target-def -I ${PROJECT_SOURCE_DIR}/lib/Target/ARM/)
+tablegen(LLVM ARMTargetParserDef.inc -gen-arm-target-def EXTRA_INCLUDES ${PROJECT_SOURCE_DIR}/lib/Target/ARM)
 
 set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/AArch64/AArch64.td)
-tablegen(LLVM AArch64TargetParserDef.inc -gen-arm-target-def -I ${PROJECT_SOURCE_DIR}/lib/Target/AArch64/)
+tablegen(LLVM AArch64TargetParserDef.inc -gen-arm-target-def EXTRA_INCLUDES ${PROJECT_SOURCE_DIR}/lib/Target/AArch64)
 
 set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/RISCV/RISCV.td)
-tablegen(LLVM RISCVTargetParserDef.inc -gen-riscv-target-def -I ${PROJECT_SOURCE_DIR}/lib/Target/RISCV/)
+tablegen(LLVM RISCVTargetParserDef.inc -gen-riscv-target-def EXTRA_INCLUDES ${PROJECT_SOURCE_DIR}/lib/Target/RISCV)
 
 set(LLVM_TARGET_DEFINITIONS ${PROJECT_SOURCE_DIR}/lib/Target/PowerPC/PPC.td)
 tablegen(LLVM PPCGenTargetFeatures.inc -gen-target-features -I${PROJECT_SOURCE_DIR}/lib/Target/PowerPC)
-- 
cgit v1.2.3


From 7f74a377d094c34eba1adde1f1edc382d01d2e5e Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 20 Jun 2025 19:10:25 +0100
Subject: [LV] Regenerate uniform_across_vf* check lines.

Re-generate check lines to reduce diff in upcoming changes.

Also filters out the code after scalar.ph:, which is dead.
---
 .../LoopVectorize/uniform_across_vf_induction1.ll  |  288 +---
 .../uniform_across_vf_induction1_and.ll            |  271 +---
 .../uniform_across_vf_induction1_div_urem.ll       |  321 ++---
 .../uniform_across_vf_induction1_lshr.ll           |  494 ++-----
 .../LoopVectorize/uniform_across_vf_induction2.ll  | 1404 +++++---------------
 5 files changed, 607 insertions(+), 2171 deletions(-)

diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll
index d9f0e180bdaa..3aad626554ce 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 2
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
 
 ; Tests for checking uniformity within a VF.
@@ -27,21 +27,6 @@ define void @ld_div1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -86,21 +71,6 @@ define void @ld_div2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -150,21 +120,6 @@ define void @ld_div3_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -220,21 +175,6 @@ define void @ld_div1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -283,21 +223,6 @@ define void @ld_div2_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -353,21 +278,6 @@ define void @ld_div3_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -423,21 +333,6 @@ define void @ld_div1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -493,21 +388,6 @@ define void @ld_div2_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -556,21 +436,6 @@ define void @ld_div3_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -614,21 +479,6 @@ define void @ld_div1_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -679,21 +529,6 @@ define void @ld_div2_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -744,21 +579,6 @@ define void @ld_div3_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -815,21 +635,6 @@ define void @ld_div1_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -879,21 +684,6 @@ define void @ld_div2_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -950,21 +740,6 @@ define void @ld_div3_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1021,21 +796,6 @@ define void @ld_div1_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1092,21 +852,6 @@ define void @ld_div2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1156,21 +901,6 @@ define void @ld_div3_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1219,22 +949,6 @@ define void @test_step_is_not_invariant(ptr %A) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ 56, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[MUL:%.*]] = mul nuw nsw i32 [[IV]], [[IV]]
-; CHECK-NEXT:    [[DIV_LHS_TRUNC:%.*]] = trunc i32 [[MUL]] to i16
-; CHECK-NEXT:    [[DIV5:%.*]] = udiv i16 [[DIV_LHS_TRUNC]], 6
-; CHECK-NEXT:    [[CONV:%.*]] = trunc i32 [[IV]] to i16
-; CHECK-NEXT:    [[IDXPROM:%.*]] = zext i16 [[DIV5]] to i64
-; CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i16, ptr [[A]], i64 [[IDXPROM]]
-; CHECK-NEXT:    store i16 [[CONV]], ptr [[ARRAYIDX]], align 2
-; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1
-; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[IV_NEXT]], 56
-; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP39:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll
index e85cf2471144..1f331a4bf973 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_and.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 2
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck %s
 
 ; Tests for checking uniformity within a VF.
@@ -13,35 +13,20 @@ define void @ld_and_neg1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[INDEX]], -1
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[TMP2]], i32 0
-; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], splat (i64 42)
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
-; CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[INDEX]], -1
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[TMP1]], i32 0
+; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw <2 x i64> [[WIDE_LOAD]], splat (i64 42)
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -71,36 +56,21 @@ define void @ld_and_neg2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = and i64 [[INDEX]], -2
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
-; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
-; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], 42
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = and i64 [[INDEX]], -2
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; CHECK-NEXT:    [[TMP3:%.*]] = add nsw i64 [[TMP2]], 42
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
-; CHECK-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -130,41 +100,26 @@ define void @ld_and_neg3_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[VEC_IND]], splat (i64 -3)
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], splat (i64 42)
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
-; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i64> [[VEC_IND]], splat (i64 -3)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <2 x i64> [[TMP8]], splat (i64 42)
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], ptr [[TMP11]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -220,21 +175,6 @@ define void @ld_and_neg1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -275,26 +215,11 @@ define void @ld_and_neg2_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    store i64 [[TMP5]], ptr [[TMP6]], align 8
 ; CHECK-NEXT:    store i64 [[TMP5]], ptr [[TMP7]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; CHECK-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
-; CHECK-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
+; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], 1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -350,21 +275,6 @@ define void @ld_and_neg1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -1
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -420,21 +330,6 @@ define void @ld_and_neg2_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -465,41 +360,26 @@ define void @ld_and_neg2_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[VEC_IND]], splat (i64 -2)
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
-; CHECK-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
-; CHECK-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], splat (i64 42)
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
-; CHECK-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = and <2 x i64> [[VEC_IND]], splat (i64 -2)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i32 0
+; CHECK-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[TMP6]], i32 1
+; CHECK-NEXT:    [[TMP9:%.*]] = add nsw <2 x i64> [[TMP8]], splat (i64 42)
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
+; CHECK-NEXT:    store <2 x i64> [[TMP9]], ptr [[TMP11]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
-; CHECK-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; CHECK-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -556,21 +436,6 @@ define void @ld_and_neg2_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -627,21 +492,6 @@ define void @ld_and_neg2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -2
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -698,21 +548,6 @@ define void @ld_and_neg3_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[AND:%.*]] = and i64 [[IV]], -3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[AND]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_div_urem.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_div_urem.ll
index 12b7353a7803..ea8831c8ab7e 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_div_urem.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_div_urem.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 2
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=8 %s -S | FileCheck %s
 
 ; Tests for checking uniformity within a VF.
@@ -15,7 +15,83 @@ define void @ld_div2_urem3_1(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv <8 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv <8 x i64> [[VEC_IND]], splat (i64 2)
+; CHECK-NEXT:    [[TMP1:%.*]] = urem <8 x i64> [[TMP0]], splat (i64 3)
+; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i64> [[TMP1]], i32 0
+; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i64> [[TMP1]], i32 1
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
+; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP1]], i32 2
+; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
+; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> [[TMP1]], i32 3
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
+; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP1]], i32 4
+; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
+; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP1]], i32 5
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP12]]
+; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP1]], i32 6
+; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP14]]
+; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP1]], i32 7
+; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP16]]
+; CHECK-NEXT:    [[TMP18:%.*]] = load i64, ptr [[TMP3]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = load i64, ptr [[TMP5]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = load i64, ptr [[TMP11]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load i64, ptr [[TMP13]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr [[TMP15]], align 8
+; CHECK-NEXT:    [[TMP25:%.*]] = load i64, ptr [[TMP17]], align 8
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <8 x i64> poison, i64 [[TMP18]], i32 0
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <8 x i64> [[TMP26]], i64 [[TMP19]], i32 1
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x i64> [[TMP27]], i64 [[TMP20]], i32 2
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x i64> [[TMP28]], i64 [[TMP21]], i32 3
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x i64> [[TMP29]], i64 [[TMP22]], i32 4
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x i64> [[TMP30]], i64 [[TMP23]], i32 5
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[TMP24]], i32 6
+; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <8 x i64> [[TMP32]], i64 [[TMP25]], i32 7
+; CHECK-NEXT:    [[TMP34:%.*]] = add nsw <8 x i64> [[TMP33]], splat (i64 42)
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i64, ptr [[TMP35]], i32 0
+; CHECK-NEXT:    store <8 x i64> [[TMP34]], ptr [[TMP36]], align 8
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
+; CHECK-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+;
+entry:
+  br label %loop
+loop:
+  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
+  %div = udiv i64 %iv, 2
+  %rem = urem i64 %div, 3
+  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %rem
+  %ld = load i64, ptr %gep_ld, align 8
+  %calc = add nsw i64 %ld, 42
+  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
+  store i64 %calc, ptr %gep_st, align 8
+  %iv_next = add nsw i64 %iv, 1
+  %cond = icmp eq i64 %iv_next, 1000
+  br i1 %cond, label %exit, label %loop
+exit:
+  ret void
+}
+
+; for (iv = 0 ; ; iv += 1) B[iv] = A[((iv++)/2)%3];
+define void @ld_div2_urem3_2(ptr noalias %A, ptr noalias %B) {
+; CHECK-LABEL: define void @ld_div2_urem3_2
+; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add nsw <8 x i64> [[VEC_IND]], splat (i64 1)
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv <8 x i64> [[TMP0]], splat (i64 2)
 ; CHECK-NEXT:    [[TMP2:%.*]] = urem <8 x i64> [[TMP1]], splat (i64 3)
 ; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i64> [[TMP2]], i32 0
 ; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
@@ -56,118 +132,10 @@ define void @ld_div2_urem3_1(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
 ; CHECK-NEXT:    [[TMP38:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK-NEXT:    br i1 [[TMP38]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 2
-; CHECK-NEXT:    [[REM:%.*]] = urem i64 [[DIV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[REM]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
-;
-entry:
-  br label %loop
-loop:
-  %iv = phi i64 [ 0, %entry ], [ %iv_next, %loop ]
-  %div = udiv i64 %iv, 2
-  %rem = urem i64 %div, 3
-  %gep_ld = getelementptr inbounds i64, ptr %A, i64 %rem
-  %ld = load i64, ptr %gep_ld, align 8
-  %calc = add nsw i64 %ld, 42
-  %gep_st = getelementptr inbounds i64, ptr %B, i64 %iv
-  store i64 %calc, ptr %gep_st, align 8
-  %iv_next = add nsw i64 %iv, 1
-  %cond = icmp eq i64 %iv_next, 1000
-  br i1 %cond, label %exit, label %loop
-exit:
-  ret void
-}
-
-; for (iv = 0 ; ; iv += 1) B[iv] = A[((iv++)/2)%3];
-define void @ld_div2_urem3_2(ptr noalias %A, ptr noalias %B) {
-; CHECK-LABEL: define void @ld_div2_urem3_2
-; CHECK-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
-; CHECK-NEXT:  entry:
-; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; CHECK:       vector.ph:
-; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
-; CHECK:       vector.body:
-; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = add nsw <8 x i64> [[VEC_IND]], splat (i64 1)
-; CHECK-NEXT:    [[TMP2:%.*]] = udiv <8 x i64> [[TMP1]], splat (i64 2)
-; CHECK-NEXT:    [[TMP3:%.*]] = urem <8 x i64> [[TMP2]], splat (i64 3)
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i64> [[TMP3]], i32 0
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP3]], i32 1
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> [[TMP3]], i32 2
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP3]], i32 3
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP3]], i32 4
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP3]], i32 5
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP3]], i32 6
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = extractelement <8 x i64> [[TMP3]], i32 7
-; CHECK-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP18]]
-; CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = load i64, ptr [[TMP9]], align 8
-; CHECK-NEXT:    [[TMP23:%.*]] = load i64, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr [[TMP13]], align 8
-; CHECK-NEXT:    [[TMP25:%.*]] = load i64, ptr [[TMP15]], align 8
-; CHECK-NEXT:    [[TMP26:%.*]] = load i64, ptr [[TMP17]], align 8
-; CHECK-NEXT:    [[TMP27:%.*]] = load i64, ptr [[TMP19]], align 8
-; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x i64> poison, i64 [[TMP20]], i32 0
-; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x i64> [[TMP28]], i64 [[TMP21]], i32 1
-; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x i64> [[TMP29]], i64 [[TMP22]], i32 2
-; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x i64> [[TMP30]], i64 [[TMP23]], i32 3
-; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[TMP24]], i32 4
-; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <8 x i64> [[TMP32]], i64 [[TMP25]], i32 5
-; CHECK-NEXT:    [[TMP34:%.*]] = insertelement <8 x i64> [[TMP33]], i64 [[TMP26]], i32 6
-; CHECK-NEXT:    [[TMP35:%.*]] = insertelement <8 x i64> [[TMP34]], i64 [[TMP27]], i32 7
-; CHECK-NEXT:    [[TMP36:%.*]] = add nsw <8 x i64> [[TMP35]], splat (i64 42)
-; CHECK-NEXT:    [[TMP37:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP38:%.*]] = getelementptr inbounds i64, ptr [[TMP37]], i32 0
-; CHECK-NEXT:    store <8 x i64> [[TMP36]], ptr [[TMP38]], align 8
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT:    [[TMP39:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
-; CHECK:       middle.block:
-; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
-; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV_NEXT]], 2
-; CHECK-NEXT:    [[REM:%.*]] = urem i64 [[DIV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[REM]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -198,65 +166,50 @@ define void @ld_div4(ptr noalias %A, ptr noalias %B) {
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_IND:%.*]] = phi <8 x i64> [ <i64 0, i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv <8 x i64> [[VEC_IND]], splat (i64 4)
-; CHECK-NEXT:    [[TMP2:%.*]] = extractelement <8 x i64> [[TMP1]], i32 0
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = extractelement <8 x i64> [[TMP1]], i32 1
-; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; CHECK-NEXT:    [[TMP6:%.*]] = extractelement <8 x i64> [[TMP1]], i32 2
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; CHECK-NEXT:    [[TMP8:%.*]] = extractelement <8 x i64> [[TMP1]], i32 3
-; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; CHECK-NEXT:    [[TMP10:%.*]] = extractelement <8 x i64> [[TMP1]], i32 4
-; CHECK-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; CHECK-NEXT:    [[TMP12:%.*]] = extractelement <8 x i64> [[TMP1]], i32 5
-; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP12]]
-; CHECK-NEXT:    [[TMP14:%.*]] = extractelement <8 x i64> [[TMP1]], i32 6
-; CHECK-NEXT:    [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP14]]
-; CHECK-NEXT:    [[TMP16:%.*]] = extractelement <8 x i64> [[TMP1]], i32 7
-; CHECK-NEXT:    [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP16]]
-; CHECK-NEXT:    [[TMP18:%.*]] = load i64, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP19:%.*]] = load i64, ptr [[TMP5]], align 8
-; CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr [[TMP7]], align 8
-; CHECK-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP9]], align 8
-; CHECK-NEXT:    [[TMP22:%.*]] = load i64, ptr [[TMP11]], align 8
-; CHECK-NEXT:    [[TMP23:%.*]] = load i64, ptr [[TMP13]], align 8
-; CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr [[TMP15]], align 8
-; CHECK-NEXT:    [[TMP25:%.*]] = load i64, ptr [[TMP17]], align 8
-; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <8 x i64> poison, i64 [[TMP18]], i32 0
-; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <8 x i64> [[TMP26]], i64 [[TMP19]], i32 1
-; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x i64> [[TMP27]], i64 [[TMP20]], i32 2
-; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x i64> [[TMP28]], i64 [[TMP21]], i32 3
-; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x i64> [[TMP29]], i64 [[TMP22]], i32 4
-; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x i64> [[TMP30]], i64 [[TMP23]], i32 5
-; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[TMP24]], i32 6
-; CHECK-NEXT:    [[TMP33:%.*]] = insertelement <8 x i64> [[TMP32]], i64 [[TMP25]], i32 7
-; CHECK-NEXT:    [[TMP34:%.*]] = add nsw <8 x i64> [[TMP33]], splat (i64 42)
-; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP36:%.*]] = getelementptr inbounds i64, ptr [[TMP35]], i32 0
-; CHECK-NEXT:    store <8 x i64> [[TMP34]], ptr [[TMP36]], align 8
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv <8 x i64> [[VEC_IND]], splat (i64 4)
+; CHECK-NEXT:    [[TMP1:%.*]] = extractelement <8 x i64> [[TMP0]], i32 0
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = extractelement <8 x i64> [[TMP0]], i32 1
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <8 x i64> [[TMP0]], i32 2
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; CHECK-NEXT:    [[TMP7:%.*]] = extractelement <8 x i64> [[TMP0]], i32 3
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; CHECK-NEXT:    [[TMP9:%.*]] = extractelement <8 x i64> [[TMP0]], i32 4
+; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; CHECK-NEXT:    [[TMP11:%.*]] = extractelement <8 x i64> [[TMP0]], i32 5
+; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]]
+; CHECK-NEXT:    [[TMP13:%.*]] = extractelement <8 x i64> [[TMP0]], i32 6
+; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP13]]
+; CHECK-NEXT:    [[TMP15:%.*]] = extractelement <8 x i64> [[TMP0]], i32 7
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP15]]
+; CHECK-NEXT:    [[TMP17:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP18:%.*]] = load i64, ptr [[TMP4]], align 8
+; CHECK-NEXT:    [[TMP19:%.*]] = load i64, ptr [[TMP6]], align 8
+; CHECK-NEXT:    [[TMP20:%.*]] = load i64, ptr [[TMP8]], align 8
+; CHECK-NEXT:    [[TMP21:%.*]] = load i64, ptr [[TMP10]], align 8
+; CHECK-NEXT:    [[TMP22:%.*]] = load i64, ptr [[TMP12]], align 8
+; CHECK-NEXT:    [[TMP23:%.*]] = load i64, ptr [[TMP14]], align 8
+; CHECK-NEXT:    [[TMP24:%.*]] = load i64, ptr [[TMP16]], align 8
+; CHECK-NEXT:    [[TMP25:%.*]] = insertelement <8 x i64> poison, i64 [[TMP17]], i32 0
+; CHECK-NEXT:    [[TMP26:%.*]] = insertelement <8 x i64> [[TMP25]], i64 [[TMP18]], i32 1
+; CHECK-NEXT:    [[TMP27:%.*]] = insertelement <8 x i64> [[TMP26]], i64 [[TMP19]], i32 2
+; CHECK-NEXT:    [[TMP28:%.*]] = insertelement <8 x i64> [[TMP27]], i64 [[TMP20]], i32 3
+; CHECK-NEXT:    [[TMP29:%.*]] = insertelement <8 x i64> [[TMP28]], i64 [[TMP21]], i32 4
+; CHECK-NEXT:    [[TMP30:%.*]] = insertelement <8 x i64> [[TMP29]], i64 [[TMP22]], i32 5
+; CHECK-NEXT:    [[TMP31:%.*]] = insertelement <8 x i64> [[TMP30]], i64 [[TMP23]], i32 6
+; CHECK-NEXT:    [[TMP32:%.*]] = insertelement <8 x i64> [[TMP31]], i64 [[TMP24]], i32 7
+; CHECK-NEXT:    [[TMP33:%.*]] = add nsw <8 x i64> [[TMP32]], splat (i64 42)
+; CHECK-NEXT:    [[TMP34:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP35:%.*]] = getelementptr inbounds i64, ptr [[TMP34]], i32 0
+; CHECK-NEXT:    store <8 x i64> [[TMP33]], ptr [[TMP35]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
 ; CHECK-NEXT:    [[VEC_IND_NEXT]] = add <8 x i64> [[VEC_IND]], splat (i64 8)
-; CHECK-NEXT:    [[TMP37:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP37]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; CHECK-NEXT:    [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP36]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 4
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[DIV]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -286,38 +239,22 @@ define void @ld_div8_urem3(ptr noalias %A, ptr noalias %B) {
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP2:%.*]] = urem i64 [[TMP1]], 3
-; CHECK-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; CHECK-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
-; CHECK-NEXT:    [[TMP5:%.*]] = add nsw i64 [[TMP4]], 42
-; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TMP5]], i64 0
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i64 [[TMP0]], 3
+; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; CHECK-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
+; CHECK-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], 42
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[TMP4]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i64> [[BROADCAST_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
-; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0
-; CHECK-NEXT:    store <8 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
+; CHECK-NEXT:    store <8 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
 ; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
-; CHECK-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
+; CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; CHECK-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; CHECK:       middle.block:
 ; CHECK-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; CHECK:       scalar.ph:
-; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    br label [[LOOP:%.*]]
-; CHECK:       loop:
-; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; CHECK-NEXT:    [[DIV:%.*]] = udiv i64 [[IV]], 8
-; CHECK-NEXT:    [[REM:%.*]] = urem i64 [[DIV]], 3
-; CHECK-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[REM]]
-; CHECK-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; CHECK-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; CHECK-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; CHECK-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; CHECK-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; CHECK-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; CHECK:       exit:
-; CHECK-NEXT:    ret void
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll
index 6b501905c33d..1f33f7a15edd 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction1_lshr.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 2
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck --check-prefixes=VF2 %s
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 %s -S | FileCheck --check-prefixes=VF4 %s
 
@@ -28,21 +28,6 @@ define void @ld_lshr0_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr0_step1_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -66,21 +51,6 @@ define void @ld_lshr0_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -125,21 +95,6 @@ define void @ld_lshr1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step1_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -150,49 +105,34 @@ define void @ld_lshr1_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i32 0
-; VF4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i32 1
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i32 2
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i32 3
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP3]], align 8
-; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; VF4-NEXT:    [[TMP18:%.*]] = add nsw <4 x i64> [[TMP17]], splat (i64 42)
-; VF4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP19]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP18]], ptr [[TMP20]], align 8
+; VF4-NEXT:    [[TMP0:%.*]] = lshr <4 x i64> [[VEC_IND]], splat (i64 1)
+; VF4-NEXT:    [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i32 0
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i32 1
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP0]], i32 2
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i32 3
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP2]], align 8
+; VF4-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; VF4-NEXT:    [[TMP17:%.*]] = add nsw <4 x i64> [[TMP16]], splat (i64 42)
+; VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP18]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP17]], ptr [[TMP19]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF4-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF4-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF4-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -222,36 +162,21 @@ define void @ld_lshr2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[TMP1:%.*]] = lshr i64 [[INDEX]], 2
-; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
-; VF2-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
-; VF2-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], 42
-; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP4]], i64 0
+; VF2-NEXT:    [[TMP0:%.*]] = lshr i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; VF2-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; VF2-NEXT:    [[TMP3:%.*]] = add nsw i64 [[TMP2]], 42
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP3]], i64 0
 ; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
-; VF2-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; VF2-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VF2-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF2-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr2_step1_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -261,36 +186,21 @@ define void @ld_lshr2_step1_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[TMP1:%.*]] = lshr i64 [[INDEX]], 2
-; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
-; VF4-NEXT:    [[TMP3:%.*]] = load i64, ptr [[TMP2]], align 8
-; VF4-NEXT:    [[TMP4:%.*]] = add nsw i64 [[TMP3]], 42
-; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP4]], i64 0
+; VF4-NEXT:    [[TMP0:%.*]] = lshr i64 [[INDEX]], 2
+; VF4-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]]
+; VF4-NEXT:    [[TMP2:%.*]] = load i64, ptr [[TMP1]], align 8
+; VF4-NEXT:    [[TMP3:%.*]] = add nsw i64 [[TMP2]], 42
+; VF4-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TMP3]], i64 0
 ; VF4-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0
-; VF4-NEXT:    store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[TMP4]], i32 0
+; VF4-NEXT:    store <4 x i64> [[BROADCAST_SPLAT]], ptr [[TMP5]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; VF4-NEXT:    [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF4-NEXT:    br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF4-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF4-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -346,21 +256,6 @@ define void @ld_lshr0_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr0_step2_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -413,21 +308,6 @@ define void @ld_lshr0_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -476,21 +356,6 @@ define void @ld_lshr1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step2_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -528,21 +393,6 @@ define void @ld_lshr1_step2_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -598,21 +448,6 @@ define void @ld_lshr0_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr0_step3_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -665,21 +500,6 @@ define void @ld_lshr0_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 0
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -735,21 +555,6 @@ define void @ld_lshr1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step3_start0_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -802,21 +607,6 @@ define void @ld_lshr1_step3_start0_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -848,41 +638,26 @@ define void @ld_lshr1_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF2-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP2:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; VF2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP3]], align 8
-; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> poison, i64 [[TMP6]], i32 0
-; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> [[TMP8]], i64 [[TMP7]], i32 1
-; VF2-NEXT:    [[TMP10:%.*]] = add nsw <2 x i64> [[TMP9]], splat (i64 42)
-; VF2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP10]], ptr [[TMP12]], align 8
+; VF2-NEXT:    [[TMP0:%.*]] = lshr <2 x i64> [[VEC_IND]], splat (i64 1)
+; VF2-NEXT:    [[TMP1:%.*]] = extractelement <2 x i64> [[TMP0]], i32 0
+; VF2-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP0]], i32 1
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = load i64, ptr [[TMP2]], align 8
+; VF2-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP7:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i32 0
+; VF2-NEXT:    [[TMP8:%.*]] = insertelement <2 x i64> [[TMP7]], i64 [[TMP6]], i32 1
+; VF2-NEXT:    [[TMP9:%.*]] = add nsw <2 x i64> [[TMP8]], splat (i64 42)
+; VF2-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP9]], ptr [[TMP11]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
-; VF2-NEXT:    br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF2-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; VF2-NEXT:    br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step1_start1_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -894,49 +669,34 @@ define void @ld_lshr1_step1_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF4-NEXT:    [[TMP1:%.*]] = lshr <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP2:%.*]] = extractelement <4 x i64> [[TMP1]], i32 0
-; VF4-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP1]], i32 1
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP1]], i32 2
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP1]], i32 3
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP3]], align 8
-; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> poison, i64 [[TMP10]], i32 0
-; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 1
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 2
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 3
-; VF4-NEXT:    [[TMP18:%.*]] = add nsw <4 x i64> [[TMP17]], splat (i64 42)
-; VF4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[TMP19]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP18]], ptr [[TMP20]], align 8
+; VF4-NEXT:    [[TMP0:%.*]] = lshr <4 x i64> [[VEC_IND]], splat (i64 1)
+; VF4-NEXT:    [[TMP1:%.*]] = extractelement <4 x i64> [[TMP0]], i32 0
+; VF4-NEXT:    [[TMP2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP0]], i32 1
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP0]], i32 2
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP0]], i32 3
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP2]], align 8
+; VF4-NEXT:    [[TMP10:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = insertelement <4 x i64> poison, i64 [[TMP9]], i32 0
+; VF4-NEXT:    [[TMP14:%.*]] = insertelement <4 x i64> [[TMP13]], i64 [[TMP10]], i32 1
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP11]], i32 2
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 3
+; VF4-NEXT:    [[TMP17:%.*]] = add nsw <4 x i64> [[TMP16]], splat (i64 42)
+; VF4-NEXT:    [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP19:%.*]] = getelementptr inbounds i64, ptr [[TMP18]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP17]], ptr [[TMP19]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
-; VF4-NEXT:    br i1 [[TMP21]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
+; VF4-NEXT:    [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
+; VF4-NEXT:    br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -986,21 +746,6 @@ define void @ld_lshr1_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step2_start1_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1039,21 +784,6 @@ define void @ld_lshr1_step2_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 993, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1110,21 +840,6 @@ define void @ld_lshr1_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr1_step3_start1_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1178,21 +893,6 @@ define void @ld_lshr1_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 1
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1249,21 +949,6 @@ define void @ld_lshr2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_lshr2_step3_start1_ind1
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1317,21 +1002,6 @@ define void @ld_lshr2_step3_start1_ind1(ptr noalias %A, ptr noalias %B) {
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[LSHR:%.*]] = lshr i64 [[IV]], 2
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[LSHR]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
diff --git a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll
index b3af23c49637..ef6255720d73 100644
--- a/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll
+++ b/llvm/test/Transforms/LoopVectorize/uniform_across_vf_induction2.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "scalar.ph:" --version 2
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 %s -S | FileCheck --check-prefix=VF2 %s
 ; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 %s -S | FileCheck --check-prefix=VF4 %s
 
@@ -13,50 +13,30 @@ define void @ld_div1_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
-; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
-; VF2-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
-; VF2-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], splat (i64 42)
-; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
+; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; VF2-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], splat (i64 42)
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step1_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -67,58 +47,38 @@ define void @ld_div1_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -152,43 +112,23 @@ define void @ld_div2_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[TMP2:%.*]] = udiv i64 [[INDEX]], 2
-; VF2-NEXT:    [[TMP3:%.*]] = udiv i64 [[INDEX]], 2
-; VF2-NEXT:    [[TMP4:%.*]] = add i64 [[TMP2]], [[TMP3]]
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP7:%.*]] = add nsw i64 [[TMP6]], 42
-; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i64 0
+; VF2-NEXT:    [[TMP0:%.*]] = udiv i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP1:%.*]] = udiv i64 [[INDEX]], 2
+; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]]
+; VF2-NEXT:    [[TMP4:%.*]] = load i64, ptr [[TMP3]], align 8
+; VF2-NEXT:    [[TMP5:%.*]] = add nsw i64 [[TMP4]], 42
+; VF2-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <2 x i64> poison, i64 [[TMP5]], i64 0
 ; VF2-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <2 x i64> [[BROADCAST_SPLATINSERT]], <2 x i64> poison, <2 x i32> zeroinitializer
-; VF2-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0
-; VF2-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP9]], align 8
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0
+; VF2-NEXT:    store <2 x i64> [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
-; VF2-NEXT:    [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT:    br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF2-NEXT:    [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF2-NEXT:    br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step1_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -199,58 +139,38 @@ define void @ld_div2_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP5:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -284,50 +204,30 @@ define void @ld_div3_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
-; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
-; VF2-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
-; VF2-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], splat (i64 42)
-; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[TMP0:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
+; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; VF2-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], splat (i64 42)
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF2-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step1_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -338,58 +238,38 @@ define void @ld_div3_step1_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP7:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -423,12 +303,12 @@ define void @ld_div1_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -447,32 +327,12 @@ define void @ld_div1_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step2_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -483,14 +343,14 @@ define void @ld_div1_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -523,32 +383,12 @@ define void @ld_div1_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -582,12 +422,12 @@ define void @ld_div2_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -606,32 +446,12 @@ define void @ld_div2_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step2_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -642,14 +462,14 @@ define void @ld_div2_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -682,32 +502,12 @@ define void @ld_div2_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP11:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -741,12 +541,12 @@ define void @ld_div3_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -765,32 +565,12 @@ define void @ld_div3_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step2_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -801,14 +581,14 @@ define void @ld_div3_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 2, i64 4, i64 6>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -841,32 +621,12 @@ define void @ld_div3_step2_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 500
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 500, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP13:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -900,12 +660,12 @@ define void @ld_div1_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -924,32 +684,12 @@ define void @ld_div1_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step3_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -960,14 +700,14 @@ define void @ld_div1_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 3, i64 6, i64 9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -1000,32 +740,12 @@ define void @ld_div1_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP15:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1059,12 +779,12 @@ define void @ld_div2_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -1083,32 +803,12 @@ define void @ld_div2_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step3_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1119,14 +819,14 @@ define void @ld_div2_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 3, i64 6, i64 9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -1159,32 +859,12 @@ define void @ld_div2_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP17:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1218,12 +898,12 @@ define void @ld_div3_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 0, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
+; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF2-NEXT:    [[TMP4:%.*]] = add <2 x i64> [[TMP2]], [[TMP3]]
 ; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP4]], i32 0
 ; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
@@ -1242,32 +922,12 @@ define void @ld_div3_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP17]], ptr [[TMP15]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step3_start0_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1278,14 +938,14 @@ define void @ld_div3_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 0, i64 3, i64 6, i64 9>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[TMP0:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF4-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP4:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
+; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF4-NEXT:    [[TMP6:%.*]] = add <4 x i64> [[TMP4]], [[TMP5]]
 ; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP6]], i32 0
 ; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
@@ -1318,32 +978,12 @@ define void @ld_div3_step3_start0_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP31]], ptr [[TMP27]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP19:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1377,51 +1017,31 @@ define void @ld_div1_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
-; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
-; VF2-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
-; VF2-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], splat (i64 42)
-; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; VF2-NEXT:    [[TMP0:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
+; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; VF2-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], splat (i64 42)
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
-; VF2-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; VF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step1_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1432,59 +1052,39 @@ define void @ld_div1_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP21:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1518,51 +1118,31 @@ define void @ld_div2_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
-; VF2-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
-; VF2-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], splat (i64 42)
-; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; VF2-NEXT:    [[TMP0:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
+; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; VF2-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], splat (i64 42)
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
-; VF2-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; VF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step1_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1573,59 +1153,39 @@ define void @ld_div2_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP23:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1659,51 +1219,31 @@ define void @ld_div3_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 2>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP2:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
-; VF2-NEXT:    [[TMP3:%.*]] = add <2 x i64> [[TMP1]], [[TMP2]]
-; VF2-NEXT:    [[TMP4:%.*]] = extractelement <2 x i64> [[TMP3]], i32 0
-; VF2-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP3]], i32 1
-; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> poison, i64 [[TMP8]], i32 0
-; VF2-NEXT:    [[TMP11:%.*]] = insertelement <2 x i64> [[TMP10]], i64 [[TMP9]], i32 1
-; VF2-NEXT:    [[TMP12:%.*]] = add nsw <2 x i64> [[TMP11]], splat (i64 42)
-; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF2-NEXT:    [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0
-; VF2-NEXT:    store <2 x i64> [[TMP12]], ptr [[TMP14]], align 8
+; VF2-NEXT:    [[TMP0:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
+; VF2-NEXT:    [[TMP1:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
+; VF2-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP0]], [[TMP1]]
+; VF2-NEXT:    [[TMP3:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
+; VF2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF2-NEXT:    [[TMP5:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
+; VF2-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF2-NEXT:    [[TMP9:%.*]] = insertelement <2 x i64> poison, i64 [[TMP7]], i32 0
+; VF2-NEXT:    [[TMP10:%.*]] = insertelement <2 x i64> [[TMP9]], i64 [[TMP8]], i32 1
+; VF2-NEXT:    [[TMP11:%.*]] = add nsw <2 x i64> [[TMP10]], splat (i64 42)
+; VF2-NEXT:    [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF2-NEXT:    [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0
+; VF2-NEXT:    store <2 x i64> [[TMP11]], ptr [[TMP13]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
-; VF2-NEXT:    [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
-; VF2-NEXT:    br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
+; VF2-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], 998
+; VF2-NEXT:    br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 999, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 998, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step1_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1714,59 +1254,39 @@ define void @ld_div3_step1_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 2, i64 3, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]]
-; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP2:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
-; VF4-NEXT:    [[TMP3:%.*]] = add <4 x i64> [[TMP1]], [[TMP2]]
-; VF4-NEXT:    [[TMP4:%.*]] = extractelement <4 x i64> [[TMP3]], i32 0
-; VF4-NEXT:    [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP4]]
-; VF4-NEXT:    [[TMP6:%.*]] = extractelement <4 x i64> [[TMP3]], i32 1
-; VF4-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
-; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP3]], i32 2
-; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
-; VF4-NEXT:    [[TMP10:%.*]] = extractelement <4 x i64> [[TMP3]], i32 3
-; VF4-NEXT:    [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP10]]
-; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP5]], align 8
-; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP7]], align 8
-; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP9]], align 8
-; VF4-NEXT:    [[TMP15:%.*]] = load i64, ptr [[TMP11]], align 8
-; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> poison, i64 [[TMP12]], i32 0
-; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 1
-; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 2
-; VF4-NEXT:    [[TMP19:%.*]] = insertelement <4 x i64> [[TMP18]], i64 [[TMP15]], i32 3
-; VF4-NEXT:    [[TMP20:%.*]] = add nsw <4 x i64> [[TMP19]], splat (i64 42)
-; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
-; VF4-NEXT:    [[TMP22:%.*]] = getelementptr inbounds i64, ptr [[TMP21]], i32 0
-; VF4-NEXT:    store <4 x i64> [[TMP20]], ptr [[TMP22]], align 8
+; VF4-NEXT:    [[TMP0:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
+; VF4-NEXT:    [[TMP1:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
+; VF4-NEXT:    [[TMP2:%.*]] = add <4 x i64> [[TMP0]], [[TMP1]]
+; VF4-NEXT:    [[TMP3:%.*]] = extractelement <4 x i64> [[TMP2]], i32 0
+; VF4-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP3]]
+; VF4-NEXT:    [[TMP5:%.*]] = extractelement <4 x i64> [[TMP2]], i32 1
+; VF4-NEXT:    [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP5]]
+; VF4-NEXT:    [[TMP7:%.*]] = extractelement <4 x i64> [[TMP2]], i32 2
+; VF4-NEXT:    [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP7]]
+; VF4-NEXT:    [[TMP9:%.*]] = extractelement <4 x i64> [[TMP2]], i32 3
+; VF4-NEXT:    [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]]
+; VF4-NEXT:    [[TMP11:%.*]] = load i64, ptr [[TMP4]], align 8
+; VF4-NEXT:    [[TMP12:%.*]] = load i64, ptr [[TMP6]], align 8
+; VF4-NEXT:    [[TMP13:%.*]] = load i64, ptr [[TMP8]], align 8
+; VF4-NEXT:    [[TMP14:%.*]] = load i64, ptr [[TMP10]], align 8
+; VF4-NEXT:    [[TMP15:%.*]] = insertelement <4 x i64> poison, i64 [[TMP11]], i32 0
+; VF4-NEXT:    [[TMP16:%.*]] = insertelement <4 x i64> [[TMP15]], i64 [[TMP12]], i32 1
+; VF4-NEXT:    [[TMP17:%.*]] = insertelement <4 x i64> [[TMP16]], i64 [[TMP13]], i32 2
+; VF4-NEXT:    [[TMP18:%.*]] = insertelement <4 x i64> [[TMP17]], i64 [[TMP14]], i32 3
+; VF4-NEXT:    [[TMP19:%.*]] = add nsw <4 x i64> [[TMP18]], splat (i64 42)
+; VF4-NEXT:    [[TMP20:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[OFFSET_IDX]]
+; VF4-NEXT:    [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[TMP20]], i32 0
+; VF4-NEXT:    store <4 x i64> [[TMP19]], ptr [[TMP21]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
-; VF4-NEXT:    [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
-; VF4-NEXT:    br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
+; VF4-NEXT:    [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], 996
+; VF4-NEXT:    br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP24:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 996, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 1
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP25:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1800,13 +1320,13 @@ define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -1825,32 +1345,12 @@ define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step2_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -1861,7 +1361,7 @@ define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 3, i64 5, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -1869,7 +1369,7 @@ define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -1902,32 +1402,12 @@ define void @ld_div1_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 496
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 993, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 496, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP27:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -1961,13 +1441,13 @@ define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -1986,32 +1466,12 @@ define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step2_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -2022,7 +1482,7 @@ define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 3, i64 5, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -2030,7 +1490,7 @@ define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -2063,32 +1523,12 @@ define void @ld_div2_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 496
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 993, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 496, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP29:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -2122,13 +1562,13 @@ define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 2
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -2147,32 +1587,12 @@ define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 4)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 498
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 498, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step2_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -2183,7 +1603,7 @@ define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 3, i64 5, i64 7>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 2
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -2191,7 +1611,7 @@ define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 4
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -2224,32 +1644,12 @@ define void @ld_div3_step2_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 8)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 496
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 993, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 496, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 2
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP31:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -2283,13 +1683,13 @@ define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 1)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 1)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -2308,32 +1708,12 @@ define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div1_step3_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -2344,7 +1724,7 @@ define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 4, i64 7, i64 10>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -2352,7 +1732,7 @@ define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 1)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 1)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 1)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -2385,32 +1765,12 @@ define void @ld_div1_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 1
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 1
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP33:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -2444,13 +1804,13 @@ define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 2)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -2469,32 +1829,12 @@ define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div2_step3_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -2505,7 +1845,7 @@ define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 4, i64 7, i64 10>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -2513,7 +1853,7 @@ define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 2)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 2)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -2546,32 +1886,12 @@ define void @ld_div2_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 2
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 2
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP35:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
@@ -2605,13 +1925,13 @@ define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2:       vector.body:
 ; VF2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[VEC_IND:%.*]] = phi <2 x i64> [ <i64 1, i64 4>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF2-NEXT:    [[VEC_IND2:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF2-NEXT:    [[VEC_IND1:%.*]] = phi <2 x i64> [ <i64 0, i64 1>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF2-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF2-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF2-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
 ; VF2-NEXT:    [[TMP2:%.*]] = add i64 [[OFFSET_IDX]], 3
 ; VF2-NEXT:    [[TMP3:%.*]] = udiv <2 x i64> [[VEC_IND]], splat (i64 3)
-; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND2]], splat (i64 3)
+; VF2-NEXT:    [[TMP4:%.*]] = udiv <2 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF2-NEXT:    [[TMP5:%.*]] = add <2 x i64> [[TMP3]], [[TMP4]]
 ; VF2-NEXT:    [[TMP6:%.*]] = extractelement <2 x i64> [[TMP5]], i32 0
 ; VF2-NEXT:    [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP6]]
@@ -2630,32 +1950,12 @@ define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF2-NEXT:    store i64 [[TMP18]], ptr [[TMP16]], align 8
 ; VF2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2
 ; VF2-NEXT:    [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 6)
-; VF2-NEXT:    [[VEC_IND_NEXT3]] = add <2 x i64> [[VEC_IND2]], splat (i64 2)
+; VF2-NEXT:    [[VEC_IND_NEXT2]] = add <2 x i64> [[VEC_IND1]], splat (i64 2)
 ; VF2-NEXT:    [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF2-NEXT:    br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; VF2:       middle.block:
 ; VF2-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF2:       scalar.ph:
-; VF2-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF2-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF2-NEXT:    br label [[LOOP:%.*]]
-; VF2:       loop:
-; VF2-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF2-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF2-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF2-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF2-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF2-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF2-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF2-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF2-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF2-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF2-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF2-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF2-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
-; VF2:       exit:
-; VF2-NEXT:    ret void
 ;
 ; VF4-LABEL: define void @ld_div3_step3_start1_ind2
 ; VF4-SAME: (ptr noalias [[A:%.*]], ptr noalias [[B:%.*]]) {
@@ -2666,7 +1966,7 @@ define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4:       vector.body:
 ; VF4-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[VEC_IND:%.*]] = phi <4 x i64> [ <i64 1, i64 4, i64 7, i64 10>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; VF4-NEXT:    [[VEC_IND2:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT3:%.*]], [[VECTOR_BODY]] ]
+; VF4-NEXT:    [[VEC_IND1:%.*]] = phi <4 x i64> [ <i64 0, i64 1, i64 2, i64 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT2:%.*]], [[VECTOR_BODY]] ]
 ; VF4-NEXT:    [[TMP0:%.*]] = mul i64 [[INDEX]], 3
 ; VF4-NEXT:    [[OFFSET_IDX:%.*]] = add i64 1, [[TMP0]]
 ; VF4-NEXT:    [[TMP1:%.*]] = add i64 [[OFFSET_IDX]], 0
@@ -2674,7 +1974,7 @@ define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    [[TMP3:%.*]] = add i64 [[OFFSET_IDX]], 6
 ; VF4-NEXT:    [[TMP4:%.*]] = add i64 [[OFFSET_IDX]], 9
 ; VF4-NEXT:    [[TMP5:%.*]] = udiv <4 x i64> [[VEC_IND]], splat (i64 3)
-; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND2]], splat (i64 3)
+; VF4-NEXT:    [[TMP6:%.*]] = udiv <4 x i64> [[VEC_IND1]], splat (i64 3)
 ; VF4-NEXT:    [[TMP7:%.*]] = add <4 x i64> [[TMP5]], [[TMP6]]
 ; VF4-NEXT:    [[TMP8:%.*]] = extractelement <4 x i64> [[TMP7]], i32 0
 ; VF4-NEXT:    [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP8]]
@@ -2707,32 +2007,12 @@ define void @ld_div3_step3_start1_ind2(ptr noalias %A, ptr noalias %B) {
 ; VF4-NEXT:    store i64 [[TMP32]], ptr [[TMP28]], align 8
 ; VF4-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
 ; VF4-NEXT:    [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 12)
-; VF4-NEXT:    [[VEC_IND_NEXT3]] = add <4 x i64> [[VEC_IND2]], splat (i64 4)
+; VF4-NEXT:    [[VEC_IND_NEXT2]] = add <4 x i64> [[VEC_IND1]], splat (i64 4)
 ; VF4-NEXT:    [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT]], 332
 ; VF4-NEXT:    br i1 [[TMP33]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]]
 ; VF4:       middle.block:
 ; VF4-NEXT:    br i1 false, label [[EXIT:%.*]], label [[SCALAR_PH]]
 ; VF4:       scalar.ph:
-; VF4-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 997, [[MIDDLE_BLOCK]] ], [ 1, [[ENTRY:%.*]] ]
-; VF4-NEXT:    [[BC_RESUME_VAL1:%.*]] = phi i64 [ 332, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ]
-; VF4-NEXT:    br label [[LOOP:%.*]]
-; VF4:       loop:
-; VF4-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[IV2:%.*]] = phi i64 [ [[BC_RESUME_VAL1]], [[SCALAR_PH]] ], [ [[IV2_NEXT:%.*]], [[LOOP]] ]
-; VF4-NEXT:    [[DIV1:%.*]] = udiv i64 [[IV]], 3
-; VF4-NEXT:    [[DIV2:%.*]] = udiv i64 [[IV2]], 3
-; VF4-NEXT:    [[ADD:%.*]] = add i64 [[DIV1]], [[DIV2]]
-; VF4-NEXT:    [[GEP_LD:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[ADD]]
-; VF4-NEXT:    [[LD:%.*]] = load i64, ptr [[GEP_LD]], align 8
-; VF4-NEXT:    [[CALC:%.*]] = add nsw i64 [[LD]], 42
-; VF4-NEXT:    [[GEP_ST:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]]
-; VF4-NEXT:    store i64 [[CALC]], ptr [[GEP_ST]], align 8
-; VF4-NEXT:    [[IV2_NEXT]] = add nsw i64 [[IV2]], 1
-; VF4-NEXT:    [[IV_NEXT]] = add nsw i64 [[IV]], 3
-; VF4-NEXT:    [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], 1000
-; VF4-NEXT:    br i1 [[COND]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP37:![0-9]+]]
-; VF4:       exit:
-; VF4-NEXT:    ret void
 ;
 entry:
   br label %loop
-- 
cgit v1.2.3


From bae48ac3c0e6f406038833199b185493a67ee08b Mon Sep 17 00:00:00 2001
From: Yijia Gu <yijiagu@google.com>
Date: Fri, 20 Jun 2025 11:14:14 -0700
Subject: [mlir][bazel] add missing deps for XeGPUTransforms

---
 utils/bazel/llvm-project-overlay/mlir/BUILD.bazel | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index 761a93ea7dfa..175d2d57d116 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -3526,6 +3526,7 @@ cc_library(
         ":Analysis",
         ":ArithDialect",
         ":ArithUtils",
+        ":ControlFlowInterfaces",
         ":DialectUtils",
         ":FunctionInterfaces",
         ":GPUDialect",
@@ -3538,6 +3539,7 @@ cc_library(
         ":MemRefDialect",
         ":Pass",
         ":SCFTransforms",
+        ":Support",
         ":TransformUtils",
         ":VectorDialect",
         ":VectorTransforms",
-- 
cgit v1.2.3


From ff6367b47071a7d80e773127e2a798c087e81ff5 Mon Sep 17 00:00:00 2001
From: Diego Caballero <dieg0ca6aller0@gmail.com>
Date: Fri, 20 Jun 2025 11:16:46 -0700
Subject: [[mlir][Vector] Add simple folders for
 `vector.from_element`/`vector.to_elements` (#144444)

This PR adds simple folders to remove no-op sequences of
`vector.from_elements` and `vector.to_elements`.
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td |  2 +
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp         | 85 ++++++++++++++++++++++++
 mlir/test/Dialect/Vector/canonicalize.mlir       | 52 +++++++++++++++
 3 files changed, 139 insertions(+)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index 85cc22ab3964..aef156c5f1d0 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -836,6 +836,7 @@ def Vector_ToElementsOp : Vector_Op<"to_elements", [
   let arguments = (ins AnyVectorOfAnyRank:$source);
   let results = (outs Variadic<AnyType>:$elements);
   let assemblyFormat = "$source attr-dict `:` type($source)";
+  let hasFolder = 1;
 }
 
 def Vector_FromElementsOp : Vector_Op<"from_elements", [
@@ -873,6 +874,7 @@ def Vector_FromElementsOp : Vector_Op<"from_elements", [
   let arguments = (ins Variadic<AnyType>:$elements);
   let results = (outs AnyFixedVectorOfAnyRank:$dest);
   let assemblyFormat = "$elements attr-dict `:` type($dest)";
+  let hasFolder = 1;
   let hasCanonicalizer = 1;
 }
 
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index e576eeac2365..6f0ac6bb5828 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -2373,10 +2373,95 @@ std::optional<SmallVector<int64_t, 4>> FMAOp::getShapeForUnroll() {
   return llvm::to_vector<4>(getVectorType().getShape());
 }
 
+//===----------------------------------------------------------------------===//
+// ToElementsOp
+//===----------------------------------------------------------------------===//
+
+/// Returns true if all the `operands` are defined by `defOp`.
+/// Otherwise, returns false.
+static bool haveSameDefiningOp(OperandRange operands, Operation *defOp) {
+  if (operands.empty())
+    return false;
+
+  return llvm::all_of(operands, [&](Value operand) {
+    Operation *currentDef = operand.getDefiningOp();
+    return currentDef == defOp;
+  });
+}
+
+/// Folds vector.to_elements(vector.from_elements(%e0, %e1, ...)) into
+/// (%e0, %e1, ...). For example:
+///
+///   %0 = vector.from_elements %a, %b, %c : vector<3xf32>
+///   %1:3 = vector.to_elements %0 : vector<3xf32>
+///   user_op %1#0, %1#1, %1#2
+///
+/// becomes:
+///
+///   user_op %a, %b, %c
+///
+static LogicalResult
+foldToElementsFromElements(ToElementsOp toElementsOp,
+                           SmallVectorImpl<OpFoldResult> &results) {
+  auto fromElementsOp =
+      toElementsOp.getSource().getDefiningOp<FromElementsOp>();
+  if (!fromElementsOp)
+    return failure();
+
+  llvm::append_range(results, fromElementsOp.getElements());
+  return success();
+}
+
+LogicalResult ToElementsOp::fold(FoldAdaptor adaptor,
+                                 SmallVectorImpl<OpFoldResult> &results) {
+  return foldToElementsFromElements(*this, results);
+}
+
 //===----------------------------------------------------------------------===//
 // FromElementsOp
 //===----------------------------------------------------------------------===//
 
+/// Folds vector.from_elements(vector.to_elements(%vector)) into %vector.
+///
+/// Case #1: Input and output vectors are the same.
+///
+///   %0:3 = vector.to_elements %a : vector<3xf32>
+///   %1 = vector.from_elements %0#0, %0#1, %0#2 : vector<3xf32>
+///   user_op %1
+///
+/// becomes:
+///
+///   user_op %a
+///
+static OpFoldResult foldFromElementsToElements(FromElementsOp fromElementsOp) {
+  OperandRange fromElemsOperands = fromElementsOp.getElements();
+  if (fromElemsOperands.empty())
+    return {};
+
+  auto toElementsOp = fromElemsOperands[0].getDefiningOp<ToElementsOp>();
+  if (!toElementsOp)
+    return {};
+
+  if (!haveSameDefiningOp(fromElemsOperands, toElementsOp))
+    return {};
+
+  // Case #1: Input and output vectors are the same. Forward the input vector.
+  Value toElementsInput = toElementsOp.getSource();
+  if (fromElementsOp.getType() == toElementsInput.getType() &&
+      llvm::equal(fromElemsOperands, toElementsOp.getResults())) {
+    return toElementsInput;
+  }
+
+  // TODO: Support cases with different input and output shapes and different
+  // number of elements.
+
+  return {};
+}
+
+OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) {
+  return foldFromElementsToElements(*this);
+}
+
 /// Rewrite a vector.from_elements into a vector.splat if all elements are the
 /// same SSA value. E.g.:
 ///
diff --git a/mlir/test/Dialect/Vector/canonicalize.mlir b/mlir/test/Dialect/Vector/canonicalize.mlir
index 6691cb52acdc..65b73375831d 100644
--- a/mlir/test/Dialect/Vector/canonicalize.mlir
+++ b/mlir/test/Dialect/Vector/canonicalize.mlir
@@ -3023,6 +3023,58 @@ func.func @extract_from_0d_splat_broadcast_regression(%a: f32, %b: vector<f32>,
 
 // -----
 
+// CHECK-LABEL: func @to_elements_from_elements_no_op(
+// CHECK-SAME:     %[[A:.*]]: f32, %[[B:.*]]: f32
+func.func @to_elements_from_elements_no_op(%a: f32, %b: f32) -> (f32, f32) {
+  // CHECK-NOT: vector.from_elements
+  // CHECK-NOT: vector.to_elements
+  %0 = vector.from_elements %b, %a : vector<2xf32>
+  %1:2 = vector.to_elements %0 : vector<2xf32>
+  // CHECK: return %[[B]], %[[A]]
+  return %1#0, %1#1 : f32, f32
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_to_elements_no_op(
+// CHECK-SAME:     %[[A:.*]]: vector<4x2xf32>
+func.func @from_elements_to_elements_no_op(%a: vector<4x2xf32>) -> vector<4x2xf32> {
+  // CHECK-NOT: vector.from_elements
+  // CHECK-NOT: vector.to_elements
+  %0:8 = vector.to_elements %a : vector<4x2xf32>
+  %1 = vector.from_elements %0#0, %0#1, %0#2, %0#3, %0#4, %0#5, %0#6, %0#7 : vector<4x2xf32>
+  // CHECK: return %[[A]]
+  return %1 : vector<4x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_to_elements_dup_elems(
+// CHECK-SAME:     %[[A:.*]]: vector<4xf32>
+func.func @from_elements_to_elements_dup_elems(%a: vector<4xf32>) -> vector<4x2xf32> {
+  // CHECK: %[[TO_EL:.*]]:4 = vector.to_elements %[[A]]
+  // CHECK: %[[FROM_EL:.*]] = vector.from_elements %[[TO_EL]]#0, %[[TO_EL]]#1, %[[TO_EL]]#2
+  %0:4 = vector.to_elements %a : vector<4xf32> // 4 elements
+  %1 = vector.from_elements %0#0, %0#1, %0#2, %0#3, %0#0, %0#1, %0#2, %0#3 : vector<4x2xf32>
+  // CHECK: return %[[FROM_EL]]
+  return %1 : vector<4x2xf32>
+}
+
+// -----
+
+// CHECK-LABEL: func @from_elements_to_elements_shuffle(
+// CHECK-SAME:     %[[A:.*]]: vector<4x2xf32>
+func.func @from_elements_to_elements_shuffle(%a: vector<4x2xf32>) -> vector<4x2xf32> {
+  // CHECK: %[[TO_EL:.*]]:8 = vector.to_elements %[[A]]
+  // CHECK: %[[FROM_EL:.*]] = vector.from_elements %[[TO_EL]]#7, %[[TO_EL]]#0, %[[TO_EL]]#6
+  %0:8 = vector.to_elements %a : vector<4x2xf32>
+  %1 = vector.from_elements %0#7, %0#0, %0#6, %0#1, %0#5, %0#2, %0#4, %0#3 : vector<4x2xf32>
+  // CHECK: return %[[FROM_EL]]
+  return %1 : vector<4x2xf32>
+}
+
+// -----
+
 // CHECK-LABEL: func @vector_insert_const_regression(
 //       CHECK:   llvm.mlir.undef
 //       CHECK:   vector.insert
-- 
cgit v1.2.3


From 521adc9fa270c1524f125f155327bf8f3743bb87 Mon Sep 17 00:00:00 2001
From: Luke Lau <luke@igalia.com>
Date: Fri, 20 Jun 2025 19:17:58 +0100
Subject: [VPlan] Use createScalarZExtOrTrunc when expanding
 expandVPWidenIntOrFpInduction

Split off from #144666
---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp         | 4 ++--
 llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index b6ca50549fa3..fbfc527ff7db 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -2608,8 +2608,8 @@ expandVPWidenIntOrFpInduction(VPWidenIntOrFpInductionRecipe *WidenIVR,
       VF = Builder.createScalarCast(Instruction::CastOps::UIToFP, VF, StepTy,
                                     DL);
     else
-      VF =
-          Builder.createScalarCast(Instruction::CastOps::Trunc, VF, StepTy, DL);
+      VF = Builder.createScalarZExtOrTrunc(VF, StepTy,
+                                           TypeInfo.inferScalarType(VF), DL);
 
     Inc = Builder.createNaryOp(MulOp, {Step, VF}, Flags);
     Inc = Builder.createNaryOp(VPInstruction::Broadcast, Inc);
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
index 7654bc9a141e..49a7fb734ade 100644
--- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
+++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll
@@ -96,8 +96,7 @@ define void @iv_expand(ptr %p, i64 %n) {
 ; CHECK-NEXT:     EMIT vp<[[BROADCAST_1:%.+]]> = broadcast ir<1>
 ; CHECK-NEXT:     EMIT vp<[[MUL:%.+]]> = mul vp<[[STEP_VECTOR]]>, vp<[[BROADCAST_1]]>
 ; CHECK-NEXT:     EMIT vp<[[INDUCTION:%.+]]> = add vp<[[BROADCAST_0]]>, vp<[[MUL]]>
-; CHECK-NEXT:     EMIT vp<[[TRUNC:%.+]]> = trunc ir<8> to i64
-; CHECK-NEXT:     EMIT vp<[[INC:%.+]]> = mul ir<1>, vp<[[TRUNC]]>
+; CHECK-NEXT:     EMIT vp<[[INC:%.+]]> = mul ir<1>, ir<8>
 ; CHECK-NEXT:     EMIT vp<[[BROADCAST_INC:%.+]]> = broadcast vp<[[INC]]>
 ; CHECK-NEXT: Successor(s): vector.body
 ; CHECK-EMPTY:
-- 
cgit v1.2.3


From 3f42c6bddd2495710331c82ce117ee7d5a58856d Mon Sep 17 00:00:00 2001
From: "Deric C." <cheung.deric@gmail.com>
Date: Fri, 20 Jun 2025 11:20:30 -0700
Subject: [DirectX] Scalarize `extractelement` and `insertelement` with dynamic
 indices (#141676)

Fixes #141136

- Implement `visitExtractElementInst` and `visitInsertElementInst` in
`DXILDataScalarizerVisitor` to scalarize `extractelement` and
`insertelement` instructions whose index operand is not a `ConstantInt`
by converting the vector to an array and then loading from the array
- Rename the `replaceVectorWithArray` helper function to
`equivalentArrayTypeFromVector`, relocate the function toward the top of
the file, and remove the unused `Ctx` parameter
---
 llvm/lib/Target/DirectX/DXILDataScalarization.cpp  | 170 ++++++++++++++++---
 .../DirectX/scalarize-dynamic-vector-index.ll      | 182 +++++++++++++++++++++
 2 files changed, 331 insertions(+), 21 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/scalarize-dynamic-vector-index.ll

diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
index 06708cec00ce..286fd0c5bb97 100644
--- a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -27,6 +27,20 @@ static const int MaxVecSize = 4;
 
 using namespace llvm;
 
+// Recursively creates an array-like version of a given vector type.
+static Type *equivalentArrayTypeFromVector(Type *T) {
+  if (auto *VecTy = dyn_cast<VectorType>(T))
+    return ArrayType::get(VecTy->getElementType(),
+                          dyn_cast<FixedVectorType>(VecTy)->getNumElements());
+  if (auto *ArrayTy = dyn_cast<ArrayType>(T)) {
+    Type *NewElementType =
+        equivalentArrayTypeFromVector(ArrayTy->getElementType());
+    return ArrayType::get(NewElementType, ArrayTy->getNumElements());
+  }
+  // If it's not a vector or array, return the original type.
+  return T;
+}
+
 class DXILDataScalarizationLegacy : public ModulePass {
 
 public:
@@ -54,8 +68,8 @@ public:
   bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
   bool visitCastInst(CastInst &CI) { return false; }
   bool visitBitCastInst(BitCastInst &BCI) { return false; }
-  bool visitInsertElementInst(InsertElementInst &IEI) { return false; }
-  bool visitExtractElementInst(ExtractElementInst &EEI) { return false; }
+  bool visitInsertElementInst(InsertElementInst &IEI);
+  bool visitExtractElementInst(ExtractElementInst &EEI);
   bool visitShuffleVectorInst(ShuffleVectorInst &SVI) { return false; }
   bool visitPHINode(PHINode &PHI) { return false; }
   bool visitLoadInst(LoadInst &LI);
@@ -65,6 +79,16 @@ public:
   friend bool findAndReplaceVectors(llvm::Module &M);
 
 private:
+  typedef std::pair<AllocaInst *, SmallVector<Value *, 4>> AllocaAndGEPs;
+  typedef SmallDenseMap<Value *, AllocaAndGEPs>
+      VectorToArrayMap; // A map from a vector-typed Value to its corresponding
+                        // AllocaInst and GEPs to each element of an array
+  VectorToArrayMap VectorAllocaMap;
+  AllocaAndGEPs createArrayFromVector(IRBuilder<> &Builder, Value *Vec,
+                                      const Twine &Name);
+  bool replaceDynamicInsertElementInst(InsertElementInst &IEI);
+  bool replaceDynamicExtractElementInst(ExtractElementInst &EEI);
+
   GlobalVariable *lookupReplacementGlobal(Value *CurrOperand);
   DenseMap<GlobalVariable *, GlobalVariable *> GlobalMap;
 };
@@ -76,6 +100,7 @@ bool DataScalarizerVisitor::visit(Function &F) {
     for (Instruction &I : make_early_inc_range(*BB))
       MadeChange |= InstVisitor::visit(I);
   }
+  VectorAllocaMap.clear();
   return MadeChange;
 }
 
@@ -90,20 +115,6 @@ DataScalarizerVisitor::lookupReplacementGlobal(Value *CurrOperand) {
   return nullptr; // Not found
 }
 
-// Recursively creates an array version of the given vector type.
-static Type *replaceVectorWithArray(Type *T, LLVMContext &Ctx) {
-  if (auto *VecTy = dyn_cast<VectorType>(T))
-    return ArrayType::get(VecTy->getElementType(),
-                          dyn_cast<FixedVectorType>(VecTy)->getNumElements());
-  if (auto *ArrayTy = dyn_cast<ArrayType>(T)) {
-    Type *NewElementType =
-        replaceVectorWithArray(ArrayTy->getElementType(), Ctx);
-    return ArrayType::get(NewElementType, ArrayTy->getNumElements());
-  }
-  // If it's not a vector or array, return the original type.
-  return T;
-}
-
 static bool isArrayOfVectors(Type *T) {
   if (ArrayType *ArrType = dyn_cast<ArrayType>(T))
     return isa<VectorType>(ArrType->getElementType());
@@ -116,8 +127,7 @@ bool DataScalarizerVisitor::visitAllocaInst(AllocaInst &AI) {
 
   ArrayType *ArrType = cast<ArrayType>(AI.getAllocatedType());
   IRBuilder<> Builder(&AI);
-  LLVMContext &Ctx = AI.getContext();
-  Type *NewType = replaceVectorWithArray(ArrType, Ctx);
+  Type *NewType = equivalentArrayTypeFromVector(ArrType);
   AllocaInst *ArrAlloca =
       Builder.CreateAlloca(NewType, nullptr, AI.getName() + ".scalarize");
   ArrAlloca->setAlignment(AI.getAlign());
@@ -173,6 +183,124 @@ bool DataScalarizerVisitor::visitStoreInst(StoreInst &SI) {
   return false;
 }
 
+DataScalarizerVisitor::AllocaAndGEPs
+DataScalarizerVisitor::createArrayFromVector(IRBuilder<> &Builder, Value *Vec,
+                                             const Twine &Name = "") {
+  // If there is already an alloca for this vector, return it
+  if (VectorAllocaMap.contains(Vec))
+    return VectorAllocaMap[Vec];
+
+  auto InsertPoint = Builder.GetInsertPoint();
+
+  // Allocate the array to hold the vector elements
+  Builder.SetInsertPointPastAllocas(Builder.GetInsertBlock()->getParent());
+  Type *ArrTy = equivalentArrayTypeFromVector(Vec->getType());
+  AllocaInst *ArrAlloca =
+      Builder.CreateAlloca(ArrTy, nullptr, Name + ".alloca");
+  const uint64_t ArrNumElems = ArrTy->getArrayNumElements();
+
+  // Create loads and stores to populate the array immediately after the
+  // original vector's defining instruction if available, else immediately after
+  // the alloca
+  if (auto *Instr = dyn_cast<Instruction>(Vec))
+    Builder.SetInsertPoint(Instr->getNextNonDebugInstruction());
+  SmallVector<Value *, 4> GEPs(ArrNumElems);
+  for (unsigned I = 0; I < ArrNumElems; ++I) {
+    Value *EE = Builder.CreateExtractElement(Vec, I, Name + ".extract");
+    GEPs[I] = Builder.CreateInBoundsGEP(
+        ArrTy, ArrAlloca, {Builder.getInt32(0), Builder.getInt32(I)},
+        Name + ".index");
+    Builder.CreateStore(EE, GEPs[I]);
+  }
+
+  VectorAllocaMap.insert({Vec, {ArrAlloca, GEPs}});
+  Builder.SetInsertPoint(InsertPoint);
+  return {ArrAlloca, GEPs};
+}
+
+/// Returns a pair of Value* with the first being a GEP into ArrAlloca using
+/// indices {0, Index}, and the second Value* being a Load of the GEP
+static std::pair<Value *, Value *>
+dynamicallyLoadArray(IRBuilder<> &Builder, AllocaInst *ArrAlloca, Value *Index,
+                     const Twine &Name = "") {
+  Type *ArrTy = ArrAlloca->getAllocatedType();
+  Value *GEP = Builder.CreateInBoundsGEP(
+      ArrTy, ArrAlloca, {Builder.getInt32(0), Index}, Name + ".index");
+  Value *Load =
+      Builder.CreateLoad(ArrTy->getArrayElementType(), GEP, Name + ".load");
+  return std::make_pair(GEP, Load);
+}
+
+bool DataScalarizerVisitor::replaceDynamicInsertElementInst(
+    InsertElementInst &IEI) {
+  IRBuilder<> Builder(&IEI);
+
+  Value *Vec = IEI.getOperand(0);
+  Value *Val = IEI.getOperand(1);
+  Value *Index = IEI.getOperand(2);
+
+  AllocaAndGEPs ArrAllocaAndGEPs =
+      createArrayFromVector(Builder, Vec, IEI.getName());
+  AllocaInst *ArrAlloca = ArrAllocaAndGEPs.first;
+  Type *ArrTy = ArrAlloca->getAllocatedType();
+  SmallVector<Value *, 4> &ArrGEPs = ArrAllocaAndGEPs.second;
+
+  auto GEPAndLoad =
+      dynamicallyLoadArray(Builder, ArrAlloca, Index, IEI.getName());
+  Value *GEP = GEPAndLoad.first;
+  Value *Load = GEPAndLoad.second;
+
+  Builder.CreateStore(Val, GEP);
+  Value *NewIEI = PoisonValue::get(Vec->getType());
+  for (unsigned I = 0; I < ArrTy->getArrayNumElements(); ++I) {
+    Value *Load = Builder.CreateLoad(ArrTy->getArrayElementType(), ArrGEPs[I],
+                                     IEI.getName() + ".load");
+    NewIEI = Builder.CreateInsertElement(NewIEI, Load, Builder.getInt32(I),
+                                         IEI.getName() + ".insert");
+  }
+
+  // Store back the original value so the Alloca can be reused for subsequent
+  // insertelement instructions on the same vector
+  Builder.CreateStore(Load, GEP);
+
+  IEI.replaceAllUsesWith(NewIEI);
+  IEI.eraseFromParent();
+  return true;
+}
+
+bool DataScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
+  // If the index is a constant then we don't need to scalarize it
+  Value *Index = IEI.getOperand(2);
+  if (isa<ConstantInt>(Index))
+    return false;
+  return replaceDynamicInsertElementInst(IEI);
+}
+
+bool DataScalarizerVisitor::replaceDynamicExtractElementInst(
+    ExtractElementInst &EEI) {
+  IRBuilder<> Builder(&EEI);
+
+  AllocaAndGEPs ArrAllocaAndGEPs =
+      createArrayFromVector(Builder, EEI.getVectorOperand(), EEI.getName());
+  AllocaInst *ArrAlloca = ArrAllocaAndGEPs.first;
+
+  auto GEPAndLoad = dynamicallyLoadArray(Builder, ArrAlloca,
+                                         EEI.getIndexOperand(), EEI.getName());
+  Value *Load = GEPAndLoad.second;
+
+  EEI.replaceAllUsesWith(Load);
+  EEI.eraseFromParent();
+  return true;
+}
+
+bool DataScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
+  // If the index is a constant then we don't need to scalarize it
+  Value *Index = EEI.getIndexOperand();
+  if (isa<ConstantInt>(Index))
+    return false;
+  return replaceDynamicExtractElementInst(EEI);
+}
+
 bool DataScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
 
   unsigned NumOperands = GEPI.getNumOperands();
@@ -197,8 +325,8 @@ bool DataScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
   return true;
 }
 
-Constant *transformInitializer(Constant *Init, Type *OrigType, Type *NewType,
-                               LLVMContext &Ctx) {
+static Constant *transformInitializer(Constant *Init, Type *OrigType,
+                                      Type *NewType, LLVMContext &Ctx) {
   // Handle ConstantAggregateZero (zero-initialized constants)
   if (isa<ConstantAggregateZero>(Init)) {
     return ConstantAggregateZero::get(NewType);
@@ -257,7 +385,7 @@ static bool findAndReplaceVectors(Module &M) {
   for (GlobalVariable &G : M.globals()) {
     Type *OrigType = G.getValueType();
 
-    Type *NewType = replaceVectorWithArray(OrigType, Ctx);
+    Type *NewType = equivalentArrayTypeFromVector(OrigType);
     if (OrigType != NewType) {
       // Create a new global variable with the updated type
       // Note: Initializer is set via transformInitializer
diff --git a/llvm/test/CodeGen/DirectX/scalarize-dynamic-vector-index.ll b/llvm/test/CodeGen/DirectX/scalarize-dynamic-vector-index.ll
new file mode 100644
index 000000000000..0eb65bd4fc75
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/scalarize-dynamic-vector-index.ll
@@ -0,0 +1,182 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes='dxil-data-scalarization' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s
+
+; Ensure that insertelement instructions have no side effects on each other
+; even in the presence of control flow
+define void @test_multiple_insert(i32 %c, i32 %i, i32 %j) {
+; CHECK-LABEL: define void @test_multiple_insert(
+; CHECK-SAME: i32 [[C:%.*]], i32 [[I:%.*]], i32 [[J:%.*]]) {
+; CHECK-NEXT:    [[V0_ALLOCA:%.*]] = alloca [2 x i32], align 4
+; CHECK-NEXT:    [[V_ALLOCA:%.*]] = alloca [2 x i32], align 4
+; CHECK-NEXT:    [[V0_0:%.*]] = insertelement <2 x i32> poison, i32 0, i32 0
+; CHECK-NEXT:    [[V0:%.*]] = insertelement <2 x i32> [[V0_0]], i32 0, i32 1
+; CHECK-NEXT:    [[V0_EXTRACT0:%.*]] = extractelement <2 x i32> [[V0]], i64 0
+; CHECK-NEXT:    [[V0_INDEX0:%.*]] = getelementptr inbounds [2 x i32], ptr [[V0_ALLOCA]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[V0_EXTRACT0]], ptr [[V0_INDEX0]], align 4
+; CHECK-NEXT:    [[V0_EXTRACT1:%.*]] = extractelement <2 x i32> [[V0]], i64 1
+; CHECK-NEXT:    [[V0_INDEX1:%.*]] = getelementptr inbounds [2 x i32], ptr [[V0_ALLOCA]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[V0_EXTRACT1]], ptr [[V0_INDEX1]], align 4
+; CHECK-NEXT:    [[COND:%.*]] = icmp eq i32 [[C]], 1
+; CHECK-NEXT:    br i1 [[COND]], label %[[IF:.*]], label %[[ELSE:.*]]
+; CHECK:       [[IF]]:
+; CHECK-NEXT:    [[V1_INDEX:%.*]] = getelementptr inbounds [2 x i32], ptr [[V0_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[V1_LOAD:%.*]] = load i32, ptr [[V1_INDEX]], align 4
+; CHECK-NEXT:    store i32 1, ptr [[V1_INDEX]], align 4
+; CHECK-NEXT:    [[V1_LOAD0:%.*]] = load i32, ptr [[V0_INDEX0]], align 4
+; CHECK-NEXT:    [[V1_INSERT0:%.*]] = insertelement <2 x i32> poison, i32 [[V1_LOAD0]], i32 0
+; CHECK-NEXT:    [[V1_LOAD1:%.*]] = load i32, ptr [[V0_INDEX1]], align 4
+; CHECK-NEXT:    [[V1_INSERT1:%.*]] = insertelement <2 x i32> [[V1_INSERT0]], i32 [[V1_LOAD1]], i32 1
+; CHECK-NEXT:    store i32 [[V1_LOAD]], ptr [[V1_INDEX]], align 4
+; CHECK-NEXT:    br label %[[EXIT:.*]]
+; CHECK:       [[ELSE]]:
+; CHECK-NEXT:    [[V2_INDEX:%.*]] = getelementptr inbounds [2 x i32], ptr [[V0_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[V2_LOAD:%.*]] = load i32, ptr [[V2_INDEX]], align 4
+; CHECK-NEXT:    store i32 2, ptr [[V2_INDEX]], align 4
+; CHECK-NEXT:    [[V2_LOAD0:%.*]] = load i32, ptr [[V0_INDEX0]], align 4
+; CHECK-NEXT:    [[V2_INSERT0:%.*]] = insertelement <2 x i32> poison, i32 [[V2_LOAD0]], i32 0
+; CHECK-NEXT:    [[V2_LOAD1:%.*]] = load i32, ptr [[V0_INDEX1]], align 4
+; CHECK-NEXT:    [[V2_INSERT1:%.*]] = insertelement <2 x i32> [[V2_INSERT0]], i32 [[V2_LOAD1]], i32 1
+; CHECK-NEXT:    store i32 [[V2_LOAD]], ptr [[V2_INDEX]], align 4
+; CHECK-NEXT:    br label %[[EXIT]]
+; CHECK:       [[EXIT]]:
+; CHECK-NEXT:    [[V:%.*]] = phi <2 x i32> [ [[V1_INSERT1]], %[[IF]] ], [ [[V2_INSERT1]], %[[ELSE]] ]
+; CHECK-NEXT:    [[V_EXTRACT:%.*]] = extractelement <2 x i32> [[V]], i64 0
+; CHECK-NEXT:    [[V_INDEX:%.*]] = getelementptr inbounds [2 x i32], ptr [[V_ALLOCA]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[V_EXTRACT]], ptr [[V_INDEX]], align 4
+; CHECK-NEXT:    [[V_EXTRACT10:%.*]] = extractelement <2 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[V_INDEX1:%.*]] = getelementptr inbounds [2 x i32], ptr [[V_ALLOCA]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[V_EXTRACT10]], ptr [[V_INDEX1]], align 4
+; CHECK-NEXT:    [[V3_INDEXJ:%.*]] = getelementptr inbounds [2 x i32], ptr [[V_ALLOCA]], i32 0, i32 [[J]]
+; CHECK-NEXT:    [[V3_LOAD:%.*]] = load i32, ptr [[V3_INDEXJ]], align 4
+; CHECK-NEXT:    store i32 3, ptr [[V3_INDEXJ]], align 4
+; CHECK-NEXT:    [[V3_LOAD0:%.*]] = load i32, ptr [[V_INDEX]], align 4
+; CHECK-NEXT:    [[V3_INSERT0:%.*]] = insertelement <2 x i32> poison, i32 [[V3_LOAD0]], i32 0
+; CHECK-NEXT:    [[V3_LOAD1:%.*]] = load i32, ptr [[V_INDEX1]], align 4
+; CHECK-NEXT:    [[V3_INSERT1:%.*]] = insertelement <2 x i32> [[V3_INSERT0]], i32 [[V3_LOAD1]], i32 1
+; CHECK-NEXT:    store i32 [[V3_LOAD]], ptr [[V3_INDEXJ]], align 4
+; CHECK-NEXT:    ret void
+;
+  %v0_0 = insertelement <2 x i32> poison, i32 0, i32 0
+  %v0 = insertelement <2 x i32> %v0_0, i32 0, i32 1
+  %cond = icmp eq i32 %c, 1
+  br i1 %cond, label %if, label %else
+if:
+  %v1 = insertelement <2 x i32> %v0, i32 1, i32 %i
+  br label %exit
+else:
+  %v2 = insertelement <2 x i32> %v0, i32 2, i32 %i
+  br label %exit
+exit:
+  %v = phi <2 x i32> [ %v1, %if ], [ %v2, %else ]
+  %v3 = insertelement <2 x i32> %v, i32 3, i32 %j
+  ret void
+}
+
+; Allocas can be reused across insert/extractelement instructions on the same vector
+define void @test_alloca_reuse(<3 x i32> %v, i32 %a, i32 %i) {
+; CHECK-LABEL: define void @test_alloca_reuse(
+; CHECK-SAME: <3 x i32> [[V:%.*]], i32 [[A:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[EE1_ALLOCA:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    [[EE1_EXTRACT:%.*]] = extractelement <3 x i32> [[V]], i64 0
+; CHECK-NEXT:    [[EE1_INDEX:%.*]] = getelementptr inbounds [3 x i32], ptr [[EE1_ALLOCA]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[EE1_EXTRACT]], ptr [[EE1_INDEX]], align 4
+; CHECK-NEXT:    [[EE1_EXTRACT1:%.*]] = extractelement <3 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[EE1_INDEX2:%.*]] = getelementptr inbounds [3 x i32], ptr [[EE1_ALLOCA]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[EE1_EXTRACT1]], ptr [[EE1_INDEX2]], align 4
+; CHECK-NEXT:    [[EE1_EXTRACT3:%.*]] = extractelement <3 x i32> [[V]], i64 2
+; CHECK-NEXT:    [[EE1_INDEX4:%.*]] = getelementptr inbounds [3 x i32], ptr [[EE1_ALLOCA]], i32 0, i32 2
+; CHECK-NEXT:    store i32 [[EE1_EXTRACT3]], ptr [[EE1_INDEX4]], align 4
+; CHECK-NEXT:    [[EE1_INDEX5:%.*]] = getelementptr inbounds [3 x i32], ptr [[EE1_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[EE1_LOAD:%.*]] = load i32, ptr [[EE1_INDEX5]], align 4
+; CHECK-NEXT:    [[IE1_DYNINDEX:%.*]] = getelementptr inbounds [3 x i32], ptr [[EE1_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[IE1_LOAD1:%.*]] = load i32, ptr [[IE1_DYNINDEX]], align 4
+; CHECK-NEXT:    store i32 [[A]], ptr [[IE1_DYNINDEX]], align 4
+; CHECK-NEXT:    [[IE1_LOAD:%.*]] = load i32, ptr [[EE1_INDEX]], align 4
+; CHECK-NEXT:    [[IE1_INSERT:%.*]] = insertelement <3 x i32> poison, i32 [[IE1_LOAD]], i32 0
+; CHECK-NEXT:    [[IE1_LOAD6:%.*]] = load i32, ptr [[EE1_INDEX2]], align 4
+; CHECK-NEXT:    [[IE1_INSERT7:%.*]] = insertelement <3 x i32> [[IE1_INSERT]], i32 [[IE1_LOAD6]], i32 1
+; CHECK-NEXT:    [[IE1_LOAD8:%.*]] = load i32, ptr [[EE1_INDEX4]], align 4
+; CHECK-NEXT:    [[IE1_INSERT9:%.*]] = insertelement <3 x i32> [[IE1_INSERT7]], i32 [[IE1_LOAD8]], i32 2
+; CHECK-NEXT:    store i32 [[IE1_LOAD1]], ptr [[IE1_DYNINDEX]], align 4
+; CHECK-NEXT:    ret void
+;
+  %ee1 = extractelement <3 x i32> %v, i32 %i
+  %ie1 = insertelement <3 x i32> %v, i32 %a, i32 %i
+  ret void
+}
+
+define float @extract_float_vec_dynamic(<4 x float> %v, i32 %i) {
+; CHECK-LABEL: define float @extract_float_vec_dynamic(
+; CHECK-SAME: <4 x float> [[V:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[EE_ALLOCA:%.*]] = alloca [4 x float], align 4
+; CHECK-NEXT:    [[EE_EXTRACT:%.*]] = extractelement <4 x float> [[V]], i64 0
+; CHECK-NEXT:    [[EE_INDEX:%.*]] = getelementptr inbounds [4 x float], ptr [[EE_ALLOCA]], i32 0, i32 0
+; CHECK-NEXT:    store float [[EE_EXTRACT]], ptr [[EE_INDEX]], align 4
+; CHECK-NEXT:    [[EE_EXTRACT1:%.*]] = extractelement <4 x float> [[V]], i64 1
+; CHECK-NEXT:    [[EE_INDEX2:%.*]] = getelementptr inbounds [4 x float], ptr [[EE_ALLOCA]], i32 0, i32 1
+; CHECK-NEXT:    store float [[EE_EXTRACT1]], ptr [[EE_INDEX2]], align 4
+; CHECK-NEXT:    [[EE_EXTRACT3:%.*]] = extractelement <4 x float> [[V]], i64 2
+; CHECK-NEXT:    [[EE_INDEX4:%.*]] = getelementptr inbounds [4 x float], ptr [[EE_ALLOCA]], i32 0, i32 2
+; CHECK-NEXT:    store float [[EE_EXTRACT3]], ptr [[EE_INDEX4]], align 4
+; CHECK-NEXT:    [[EE_EXTRACT5:%.*]] = extractelement <4 x float> [[V]], i64 3
+; CHECK-NEXT:    [[EE_INDEX6:%.*]] = getelementptr inbounds [4 x float], ptr [[EE_ALLOCA]], i32 0, i32 3
+; CHECK-NEXT:    store float [[EE_EXTRACT5]], ptr [[EE_INDEX6]], align 4
+; CHECK-NEXT:    [[EE_INDEX7:%.*]] = getelementptr inbounds [4 x float], ptr [[EE_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[EE_LOAD:%.*]] = load float, ptr [[EE_INDEX7]], align 4
+; CHECK-NEXT:    ret float [[EE_LOAD]]
+;
+  %ee = extractelement <4 x float> %v, i32 %i
+  ret float %ee
+}
+
+define <3 x i32> @insert_i32_vec_dynamic(<3 x i32> %v, i32 %a, i32 %i) {
+; CHECK-LABEL: define <3 x i32> @insert_i32_vec_dynamic(
+; CHECK-SAME: <3 x i32> [[V:%.*]], i32 [[A:%.*]], i32 [[I:%.*]]) {
+; CHECK-NEXT:    [[IE_ALLOCA:%.*]] = alloca [3 x i32], align 4
+; CHECK-NEXT:    [[IE_EXTRACT:%.*]] = extractelement <3 x i32> [[V]], i64 0
+; CHECK-NEXT:    [[IE_INDEX:%.*]] = getelementptr inbounds [3 x i32], ptr [[IE_ALLOCA]], i32 0, i32 0
+; CHECK-NEXT:    store i32 [[IE_EXTRACT]], ptr [[IE_INDEX]], align 4
+; CHECK-NEXT:    [[IE_EXTRACT1:%.*]] = extractelement <3 x i32> [[V]], i64 1
+; CHECK-NEXT:    [[IE_INDEX2:%.*]] = getelementptr inbounds [3 x i32], ptr [[IE_ALLOCA]], i32 0, i32 1
+; CHECK-NEXT:    store i32 [[IE_EXTRACT1]], ptr [[IE_INDEX2]], align 4
+; CHECK-NEXT:    [[IE_EXTRACT3:%.*]] = extractelement <3 x i32> [[V]], i64 2
+; CHECK-NEXT:    [[IE_INDEX4:%.*]] = getelementptr inbounds [3 x i32], ptr [[IE_ALLOCA]], i32 0, i32 2
+; CHECK-NEXT:    store i32 [[IE_EXTRACT3]], ptr [[IE_INDEX4]], align 4
+; CHECK-NEXT:    [[IE_DYNINDEX:%.*]] = getelementptr inbounds [3 x i32], ptr [[IE_ALLOCA]], i32 0, i32 [[I]]
+; CHECK-NEXT:    [[IE_LOAD1:%.*]] = load i32, ptr [[IE_DYNINDEX]], align 4
+; CHECK-NEXT:    store i32 [[A]], ptr [[IE_DYNINDEX]], align 4
+; CHECK-NEXT:    [[IE_LOAD:%.*]] = load i32, ptr [[IE_INDEX]], align 4
+; CHECK-NEXT:    [[IE_INSERT:%.*]] = insertelement <3 x i32> poison, i32 [[IE_LOAD]], i32 0
+; CHECK-NEXT:    [[IE_LOAD5:%.*]] = load i32, ptr [[IE_INDEX2]], align 4
+; CHECK-NEXT:    [[IE_INSERT6:%.*]] = insertelement <3 x i32> [[IE_INSERT]], i32 [[IE_LOAD5]], i32 1
+; CHECK-NEXT:    [[IE_LOAD7:%.*]] = load i32, ptr [[IE_INDEX4]], align 4
+; CHECK-NEXT:    [[IE_INSERT8:%.*]] = insertelement <3 x i32> [[IE_INSERT6]], i32 [[IE_LOAD7]], i32 2
+; CHECK-NEXT:    store i32 [[IE_LOAD1]], ptr [[IE_DYNINDEX]], align 4
+; CHECK-NEXT:    ret <3 x i32> [[IE_INSERT8]]
+;
+  %ie = insertelement <3 x i32> %v, i32 %a, i32 %i
+  ret <3 x i32> %ie
+}
+
+; An extractelement with a constant index should not be converted to array form
+define i16 @extract_i16_vec_constant(<4 x i16> %v) {
+; CHECK-LABEL: define i16 @extract_i16_vec_constant(
+; CHECK-SAME: <4 x i16> [[V:%.*]]) {
+; CHECK-NEXT:    [[EE:%.*]] = extractelement <4 x i16> [[V]], i32 1
+; CHECK-NEXT:    ret i16 [[EE]]
+;
+  %ee = extractelement <4 x i16> %v, i32 1
+  ret i16 %ee
+}
+
+; An insertelement with a constant index should not be converted to array form
+define <2 x half> @insert_half_vec_constant(<2 x half> %v, half %a) {
+; CHECK-LABEL: define <2 x half> @insert_half_vec_constant(
+; CHECK-SAME: <2 x half> [[V:%.*]], half [[A:%.*]]) {
+; CHECK-NEXT:    [[IE:%.*]] = insertelement <2 x half> [[V]], half [[A]], i32 1
+; CHECK-NEXT:    ret <2 x half> [[IE]]
+;
+  %ie = insertelement <2 x half> %v, half %a, i32 1
+  ret <2 x half> %ie
+}
+
-- 
cgit v1.2.3


From 925dbc798828d78e5300972dfdabb89955216229 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <Krzysztof.Parzyszek@amd.com>
Date: Fri, 20 Jun 2025 13:18:38 -0500
Subject: [flang][OpenMP] Fix namespace nesting after PR144960

Newly introduced Atomic.cpp fails to compile on its own, but somehow
compiles fine in the build. Maybe it's because PCH, but it needs to be
fixed nevertheless.
---
 flang/lib/Lower/OpenMP/Atomic.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/flang/lib/Lower/OpenMP/Atomic.h b/flang/lib/Lower/OpenMP/Atomic.h
index b83773b11300..96db4d7e90c8 100644
--- a/flang/lib/Lower/OpenMP/Atomic.h
+++ b/flang/lib/Lower/OpenMP/Atomic.h
@@ -12,16 +12,16 @@ namespace Fortran {
 namespace lower {
 class AbstractConverter;
 class SymMap;
+
+namespace pft {
+struct Evaluation;
+}
 } // namespace lower
 
 namespace parser {
 struct OpenMPAtomicConstruct;
 }
 
-namespace pft {
-struct Evaluation;
-}
-
 namespace semantics {
 class SemanticsContext;
 }
-- 
cgit v1.2.3


From f159774352c37a75829c04febb89f141175fc2bf Mon Sep 17 00:00:00 2001
From: Fabian Mora <fmora.dev@gmail.com>
Date: Fri, 20 Jun 2025 14:23:39 -0400
Subject: [mlir][core|ptr] Add `PtrLikeTypeInterface` and casting ops to the
 `ptr` dialect (#137469)

This patch adds the `PtrLikeTypeInterface` type interface to identify
pointer-like types. This interface is defined as:

```
A ptr-like type represents an object storing a memory address. This object
is constituted by:
- A memory address called the base pointer. This pointer is treated as a
  bag of bits without any assumed structure. The bit-width of the base
  pointer must be a compile-time constant. However, the bit-width may remain
  opaque or unavailable during transformations that do not depend on the
  base pointer. Finally, it is considered indivisible in the sense that as
  a `PtrLikeTypeInterface` value, it has no metadata.
- Optional metadata about the pointer. For example, the size of the  memory
  region associated with the pointer.

Furthermore, all ptr-like types have two properties:
- The memory space associated with the address held by the pointer.
- An optional element type. If the element type is not specified, the
  pointer is considered opaque.
```

This patch adds this interface to `!ptr.ptr` and the `memref` type.

Furthermore, this patch adds necessary ops and type to handle casting
between `!ptr.ptr` and ptr-like types.

First, it defines the `!ptr.ptr_metadata` type. An opaque type to
represent the metadata of a ptr-like type. The rationale behind adding
this type, is that at high-level the metadata of a type like `memref`
cannot be specified, as its structure is tied to its lowering.

The `ptr.get_metadata` operation was added to extract the opaque pointer
metadata. The concrete structure of the metadata is only known when the
op is lowered.

Finally, this patch adds the `ptr.from_ptr` and `ptr.to_ptr` operations.
Allowing to cast back and forth between `!ptr.ptr` and ptr-like types.

```mlir
func.func @func(%mr: memref<f32, #ptr.generic_space>) -> memref<f32, #ptr.generic_space> {
  %ptr = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
  %mda = ptr.get_metadata %mr : memref<f32, #ptr.generic_space>
  %res = ptr.from_ptr %ptr metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
  return %res : memref<f32, #ptr.generic_space>
}
```

It's future work to replace and remove the `bare-ptr-convention` through
the use of these ops.

---------

Co-authored-by: Mehdi Amini <joker.eph@gmail.com>
---
 mlir/include/mlir/Dialect/Ptr/IR/PtrDialect.td |  50 ++++++++++++
 mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td     | 100 ++++++++++++++++++++++-
 mlir/include/mlir/IR/BuiltinTypeInterfaces.td  |  53 +++++++++++++
 mlir/include/mlir/IR/BuiltinTypes.h            |  18 ++++-
 mlir/include/mlir/IR/BuiltinTypes.td           |   2 +
 mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp         |  80 +++++++++++++++++++
 mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp           |  12 +++
 mlir/lib/IR/BuiltinTypes.cpp                   |  14 ++++
 mlir/test/Dialect/Ptr/canonicalize.mlir        | 106 +++++++++++++++++++++++++
 mlir/test/Dialect/Ptr/invalid.mlir             |  16 ++++
 mlir/test/Dialect/Ptr/ops.mlir                 |  10 +++
 11 files changed, 458 insertions(+), 3 deletions(-)
 create mode 100644 mlir/test/Dialect/Ptr/invalid.mlir

diff --git a/mlir/include/mlir/Dialect/Ptr/IR/PtrDialect.td b/mlir/include/mlir/Dialect/Ptr/IR/PtrDialect.td
index 73b2a0857cef..7407d74ce3a8 100644
--- a/mlir/include/mlir/Dialect/Ptr/IR/PtrDialect.td
+++ b/mlir/include/mlir/Dialect/Ptr/IR/PtrDialect.td
@@ -37,6 +37,7 @@ class Ptr_Type<string name, string typeMnemonic, list<Trait> traits = []>
 
 def Ptr_PtrType : Ptr_Type<"Ptr", "ptr", [
     MemRefElementTypeInterface,
+    PtrLikeTypeInterface,
     VectorElementTypeInterface,
     DeclareTypeInterfaceMethods<DataLayoutTypeInterface, [
       "areCompatible", "getIndexBitwidth", "verifyEntries",
@@ -63,6 +64,55 @@ def Ptr_PtrType : Ptr_Type<"Ptr", "ptr", [
       return $_get(memorySpace.getContext(), memorySpace);
     }]>
   ];
+  let extraClassDeclaration = [{
+    // `PtrLikeTypeInterface` interface methods.
+    /// Returns `Type()` as this pointer type is opaque.
+    Type getElementType() const {
+      return Type();
+    }
+    /// Clones the pointer with specified memory space or returns failure
+    /// if an `elementType` was specified or if the memory space doesn't
+    /// implement `MemorySpaceAttrInterface`.
+    FailureOr<PtrLikeTypeInterface> clonePtrWith(Attribute memorySpace,
+      std::optional<Type> elementType) const {
+      if (elementType)
+        return failure();
+      if (auto ms = dyn_cast<MemorySpaceAttrInterface>(memorySpace))
+        return cast<PtrLikeTypeInterface>(get(ms));
+      return failure();
+    }
+    /// `!ptr.ptr` types are seen as ptr-like objects with no metadata.
+    bool hasPtrMetadata() const {
+      return false;
+    }
+  }];
+}
+
+def Ptr_PtrMetadata : Ptr_Type<"PtrMetadata", "ptr_metadata"> {
+  let summary = "Pointer metadata type";
+  let description = [{
+    The `ptr_metadata` type represents an opaque-view of the metadata associated
+    with a `ptr-like` object type.
+
+    Note: It's a verification error to construct a `ptr_metadata` type using a
+    `ptr-like` type with no metadata.
+
+    Example:
+
+    ```mlir
+    // The metadata associated with a `memref` type.
+    !ptr.ptr_metadata<memref<f32>>
+    ```
+  }];
+  let parameters = (ins "PtrLikeTypeInterface":$type);
+  let assemblyFormat = "`<` $type `>`";
+  let builders = [
+    TypeBuilderWithInferredContext<(ins
+      "PtrLikeTypeInterface":$ptrLike), [{
+      return $_get(ptrLike.getContext(), ptrLike);
+    }]>
+  ];
+  let genVerifyDecl = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
index 791b95ad3559..1523762efc18 100644
--- a/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
+++ b/mlir/include/mlir/Dialect/Ptr/IR/PtrOps.td
@@ -17,6 +17,72 @@ include "mlir/Interfaces/SideEffectInterfaces.td"
 include "mlir/Interfaces/ViewLikeInterface.td"
 include "mlir/IR/OpAsmInterface.td"
 
+//===----------------------------------------------------------------------===//
+// FromPtrOp
+//===----------------------------------------------------------------------===//
+
+def Ptr_FromPtrOp : Pointer_Op<"from_ptr", [
+    Pure, OptionalTypesMatchWith<"metadata type", "result", "metadata",
+            "PtrMetadataType::get(cast<PtrLikeTypeInterface>($_self))">
+  ]> {
+  let summary = "Casts a `!ptr.ptr` value to a ptr-like value.";
+  let description = [{
+    The `from_ptr` operation casts a `ptr` value to a ptr-like object. It's
+    important to note that:
+    - The ptr-like object cannot be a `!ptr.ptr`.
+    - The memory-space of both the `ptr` and ptr-like object must match.
+    - The cast is Pure (no UB and side-effect free).
+
+    The optional `metadata` operand exists to provide any ptr-like metadata
+    that might be required to perform the cast.
+
+    Example:
+
+    ```mlir
+    %typed_ptr = ptr.from_ptr %ptr : !ptr.ptr<#ptr.generic_space> -> !my.ptr<f32, #ptr.generic_space>
+    %memref = ptr.from_ptr %ptr metadata %md : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  
+    // Cast the `%ptr` to a memref without utilizing metadata.
+    %memref = ptr.from_ptr %ptr : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+    ```
+  }];
+
+  let arguments = (ins Ptr_PtrType:$ptr, Optional<Ptr_PtrMetadata>:$metadata);
+  let results = (outs PtrLikeTypeInterface:$result);
+  let assemblyFormat = [{
+    $ptr (`metadata` $metadata^)? attr-dict `:` type($ptr) `->` type($result)
+  }];
+  let hasFolder = 1;
+  let hasVerifier = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// GetMetadataOp
+//===----------------------------------------------------------------------===//
+
+def Ptr_GetMetadataOp : Pointer_Op<"get_metadata", [
+    Pure, TypesMatchWith<"metadata type", "ptr", "result",
+            "PtrMetadataType::get(cast<PtrLikeTypeInterface>($_self))">
+  ]> {
+  let summary = "SSA value representing pointer metadata.";
+  let description = [{
+    The `get_metadata` operation produces an opaque value that encodes the
+    metadata of the ptr-like type.
+
+    Example:
+
+    ```mlir
+    %metadata = ptr.get_metadata %memref : memref<?x?xf32>
+    ```
+  }];
+
+  let arguments = (ins PtrLikeTypeInterface:$ptr);
+  let results = (outs Ptr_PtrMetadata:$result);
+  let assemblyFormat = [{
+    $ptr attr-dict `:` type($ptr)
+  }];
+}
+
 //===----------------------------------------------------------------------===//
 // PtrAddOp
 //===----------------------------------------------------------------------===//
@@ -32,8 +98,8 @@ def Ptr_PtrAddOp : Pointer_Op<"ptr_add", [
     Example:
 
     ```mlir
-    %x_off  = ptr.ptr_add %x, %off : !ptr.ptr<0>, i32
-    %x_off0 = ptr.ptr_add nusw %x, %off : !ptr.ptr<0>, i32
+    %x_off  = ptr.ptr_add %x, %off : !ptr.ptr<#ptr.generic_space>, i32
+    %x_off0 = ptr.ptr_add nusw %x, %off : !ptr.ptr<#ptr.generic_space>, i32
     ```
   }];
 
@@ -52,6 +118,36 @@ def Ptr_PtrAddOp : Pointer_Op<"ptr_add", [
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// ToPtrOp
+//===----------------------------------------------------------------------===//
+
+def Ptr_ToPtrOp : Pointer_Op<"to_ptr", [Pure]> {
+  let summary = "Casts a ptr-like value to a `!ptr.ptr` value.";
+  let description = [{
+    The `to_ptr` operation casts a ptr-like object to a `!ptr.ptr`. It's
+    important to note that:
+    - The ptr-like object cannot be a `!ptr.ptr`.
+    - The memory-space of both the `ptr` and ptr-like object must match.
+    - The cast is side-effect free.
+
+    Example:
+
+    ```mlir
+    %ptr0 = ptr.to_ptr %my_ptr : !my.ptr<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+    %ptr1 = ptr.to_ptr %memref : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+    ```
+  }];
+
+  let arguments = (ins PtrLikeTypeInterface:$ptr);
+  let results = (outs Ptr_PtrType:$result);
+  let assemblyFormat = [{
+    $ptr attr-dict `:` type($ptr) `->` type($result)
+  }];
+  let hasFolder = 1;
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // TypeOffsetOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinTypeInterfaces.td b/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
index 4a4f818b46c5..367aeb6ac512 100644
--- a/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
+++ b/mlir/include/mlir/IR/BuiltinTypeInterfaces.td
@@ -110,6 +110,59 @@ def MemRefElementTypeInterface : TypeInterface<"MemRefElementTypeInterface"> {
   }];
 }
 
+//===----------------------------------------------------------------------===//
+// PtrLikeTypeInterface
+//===----------------------------------------------------------------------===//
+
+def PtrLikeTypeInterface : TypeInterface<"PtrLikeTypeInterface"> {
+  let cppNamespace = "::mlir";
+  let description = [{
+    A ptr-like type represents an object storing a memory address. This object
+    is constituted by:
+    - A memory address called the base pointer. This pointer is treated as a
+      bag of bits without any assumed structure. The bit-width of the base
+      pointer must be a compile-time constant. However, the bit-width may remain
+      opaque or unavailable during transformations that do not depend on the
+      base pointer. Finally, it is considered indivisible in the sense that as
+      a `PtrLikeTypeInterface` value, it has no metadata.
+    - Optional metadata about the pointer. For example, the size of the  memory
+      region associated with the pointer.
+
+    Furthermore, all ptr-like types have two properties:
+    - The memory space associated with the address held by the pointer.
+    - An optional element type. If the element type is not specified, the
+      pointer is considered opaque.
+  }];
+  let methods = [
+    InterfaceMethod<[{
+      Returns the memory space of this ptr-like type.
+    }],
+    "::mlir::Attribute", "getMemorySpace">,
+    InterfaceMethod<[{
+      Returns the element type of this ptr-like type. Note: this method can
+      return `::mlir::Type()`, in which case the pointer is considered opaque.
+    }],
+    "::mlir::Type", "getElementType">,
+    InterfaceMethod<[{
+      Returns whether this ptr-like type has non-empty metadata.
+    }],
+    "bool", "hasPtrMetadata">,
+    InterfaceMethod<[{
+      Returns a clone of this type with the given memory space and element type,
+      or `failure` if the type cannot be cloned with the specified arguments.
+      If the pointer is opaque and `elementType` is not `std::nullopt` the
+      method will return `failure`.
+
+      If no `elementType` is provided and ptr is not opaque, the `elementType`
+      of this type is used.
+    }],
+    "::llvm::FailureOr<::mlir::PtrLikeTypeInterface>", "clonePtrWith", (ins
+      "::mlir::Attribute":$memorySpace,
+      "::std::optional<::mlir::Type>":$elementType
+    )>
+  ];
+}
+
 //===----------------------------------------------------------------------===//
 // ShapedType
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinTypes.h b/mlir/include/mlir/IR/BuiltinTypes.h
index df1e02732617..86ec5c43970b 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.h
+++ b/mlir/include/mlir/IR/BuiltinTypes.h
@@ -99,7 +99,9 @@ public:
 /// Note: This class attaches the ShapedType trait to act as a mixin to
 ///       provide many useful utility functions. This inheritance has no effect
 ///       on derived memref types.
-class BaseMemRefType : public Type, public ShapedType::Trait<BaseMemRefType> {
+class BaseMemRefType : public Type,
+                       public PtrLikeTypeInterface::Trait<BaseMemRefType>,
+                       public ShapedType::Trait<BaseMemRefType> {
 public:
   using Type::Type;
 
@@ -117,6 +119,12 @@ public:
   BaseMemRefType cloneWith(std::optional<ArrayRef<int64_t>> shape,
                            Type elementType) const;
 
+  /// Clone this type with the given memory space and element type. If the
+  /// provided element type is `std::nullopt`, the current element type of the
+  /// type is used.
+  FailureOr<PtrLikeTypeInterface>
+  clonePtrWith(Attribute memorySpace, std::optional<Type> elementType) const;
+
   // Make sure that base class overloads are visible.
   using ShapedType::Trait<BaseMemRefType>::clone;
 
@@ -141,8 +149,16 @@ public:
   /// New `Attribute getMemorySpace()` method should be used instead.
   unsigned getMemorySpaceAsInt() const;
 
+  /// Returns that this ptr-like object has non-empty ptr metadata.
+  bool hasPtrMetadata() const { return true; }
+
   /// Allow implicit conversion to ShapedType.
   operator ShapedType() const { return llvm::cast<ShapedType>(*this); }
+
+  /// Allow implicit conversion to PtrLikeTypeInterface.
+  operator PtrLikeTypeInterface() const {
+    return llvm::cast<PtrLikeTypeInterface>(*this);
+  }
 };
 
 } // namespace mlir
diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td
index d5663bcbf6a5..55d64d663f7e 100644
--- a/mlir/include/mlir/IR/BuiltinTypes.td
+++ b/mlir/include/mlir/IR/BuiltinTypes.td
@@ -562,6 +562,7 @@ def Builtin_Integer : Builtin_Type<"Integer", "integer",
 //===----------------------------------------------------------------------===//
 
 def Builtin_MemRef : Builtin_Type<"MemRef", "memref", [
+    PtrLikeTypeInterface,
     ShapedTypeInterface
   ], "BaseMemRefType"> {
   let summary = "Shaped reference to a region of memory";
@@ -1143,6 +1144,7 @@ def Builtin_Tuple : Builtin_Type<"Tuple", "tuple"> {
 //===----------------------------------------------------------------------===//
 
 def Builtin_UnrankedMemRef : Builtin_Type<"UnrankedMemRef", "unranked_memref", [
+    PtrLikeTypeInterface,
     ShapedTypeInterface
   ], "BaseMemRefType"> {
   let summary = "Shaped reference, with unknown rank, to a region of memory";
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
index c21783011452..c48814450812 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrDialect.cpp
@@ -41,6 +41,52 @@ void PtrDialect::initialize() {
       >();
 }
 
+//===----------------------------------------------------------------------===//
+// FromPtrOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult FromPtrOp::fold(FoldAdaptor adaptor) {
+  // Fold the pattern:
+  // %ptr = ptr.to_ptr %v : type -> ptr
+  // (%mda = ptr.get_metadata %v : type)?
+  // %val = ptr.from_ptr %ptr (metadata %mda)? : ptr -> type
+  // To:
+  // %val -> %v
+  Value ptrLike;
+  FromPtrOp fromPtr = *this;
+  while (fromPtr != nullptr) {
+    auto toPtr = dyn_cast_or_null<ToPtrOp>(fromPtr.getPtr().getDefiningOp());
+    // Cannot fold if it's not a `to_ptr` op or the initial and final types are
+    // different.
+    if (!toPtr || toPtr.getPtr().getType() != fromPtr.getType())
+      return ptrLike;
+    Value md = fromPtr.getMetadata();
+    // If the type has trivial metadata fold.
+    if (!fromPtr.getType().hasPtrMetadata()) {
+      ptrLike = toPtr.getPtr();
+    } else if (md) {
+      // Fold if the metadata can be verified to be equal.
+      if (auto mdOp = dyn_cast_or_null<GetMetadataOp>(md.getDefiningOp());
+          mdOp && mdOp.getPtr() == toPtr.getPtr())
+        ptrLike = toPtr.getPtr();
+    }
+    // Check for a sequence of casts.
+    fromPtr = dyn_cast_or_null<FromPtrOp>(ptrLike ? ptrLike.getDefiningOp()
+                                                  : nullptr);
+  }
+  return ptrLike;
+}
+
+LogicalResult FromPtrOp::verify() {
+  if (isa<PtrType>(getType()))
+    return emitError() << "the result type cannot be `!ptr.ptr`";
+  if (getType().getMemorySpace() != getPtr().getType().getMemorySpace()) {
+    return emitError()
+           << "expected the input and output to have the same memory space";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // PtrAddOp
 //===----------------------------------------------------------------------===//
@@ -55,6 +101,40 @@ OpFoldResult PtrAddOp::fold(FoldAdaptor adaptor) {
   return nullptr;
 }
 
+//===----------------------------------------------------------------------===//
+// ToPtrOp
+//===----------------------------------------------------------------------===//
+
+OpFoldResult ToPtrOp::fold(FoldAdaptor adaptor) {
+  // Fold the pattern:
+  // %val = ptr.from_ptr %p (metadata ...)? : ptr -> type
+  // %ptr = ptr.to_ptr %val : type -> ptr
+  // To:
+  // %ptr -> %p
+  Value ptr;
+  ToPtrOp toPtr = *this;
+  while (toPtr != nullptr) {
+    auto fromPtr = dyn_cast_or_null<FromPtrOp>(toPtr.getPtr().getDefiningOp());
+    // Cannot fold if it's not a `from_ptr` op.
+    if (!fromPtr)
+      return ptr;
+    ptr = fromPtr.getPtr();
+    // Check for chains of casts.
+    toPtr = dyn_cast_or_null<ToPtrOp>(ptr.getDefiningOp());
+  }
+  return ptr;
+}
+
+LogicalResult ToPtrOp::verify() {
+  if (isa<PtrType>(getPtr().getType()))
+    return emitError() << "the input value cannot be of type `!ptr.ptr`";
+  if (getType().getMemorySpace() != getPtr().getType().getMemorySpace()) {
+    return emitError()
+           << "expected the input and output to have the same memory space";
+  }
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // TypeOffsetOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp b/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
index cab9ca11e679..7ad2a6bc4c80 100644
--- a/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
+++ b/mlir/lib/Dialect/Ptr/IR/PtrTypes.cpp
@@ -151,3 +151,15 @@ LogicalResult PtrType::verifyEntries(DataLayoutEntryListRef entries,
   }
   return success();
 }
+
+//===----------------------------------------------------------------------===//
+// Pointer metadata
+//===----------------------------------------------------------------------===//
+
+LogicalResult
+PtrMetadataType::verify(function_ref<InFlightDiagnostic()> emitError,
+                        PtrLikeTypeInterface type) {
+  if (!type.hasPtrMetadata())
+    return emitError() << "the ptr-like type has no metadata";
+  return success();
+}
diff --git a/mlir/lib/IR/BuiltinTypes.cpp b/mlir/lib/IR/BuiltinTypes.cpp
index d47e360e9dc1..97bab479c79b 100644
--- a/mlir/lib/IR/BuiltinTypes.cpp
+++ b/mlir/lib/IR/BuiltinTypes.cpp
@@ -376,6 +376,20 @@ BaseMemRefType BaseMemRefType::cloneWith(std::optional<ArrayRef<int64_t>> shape,
   return builder;
 }
 
+FailureOr<PtrLikeTypeInterface>
+BaseMemRefType::clonePtrWith(Attribute memorySpace,
+                             std::optional<Type> elementType) const {
+  Type eTy = elementType ? *elementType : getElementType();
+  if (llvm::dyn_cast<UnrankedMemRefType>(*this))
+    return cast<PtrLikeTypeInterface>(
+        UnrankedMemRefType::get(eTy, memorySpace));
+
+  MemRefType::Builder builder(llvm::cast<MemRefType>(*this));
+  builder.setElementType(eTy);
+  builder.setMemorySpace(memorySpace);
+  return cast<PtrLikeTypeInterface>(static_cast<MemRefType>(builder));
+}
+
 MemRefType BaseMemRefType::clone(::llvm::ArrayRef<int64_t> shape,
                                  Type elementType) const {
   return ::llvm::cast<MemRefType>(cloneWith(shape, elementType));
diff --git a/mlir/test/Dialect/Ptr/canonicalize.mlir b/mlir/test/Dialect/Ptr/canonicalize.mlir
index ad363d554f24..e50cd1b76caf 100644
--- a/mlir/test/Dialect/Ptr/canonicalize.mlir
+++ b/mlir/test/Dialect/Ptr/canonicalize.mlir
@@ -13,3 +13,109 @@ func.func @zero_offset(%ptr: !ptr.ptr<#ptr.generic_space>) -> !ptr.ptr<#ptr.gene
   %res0 = ptr.ptr_add %ptr, %off : !ptr.ptr<#ptr.generic_space>, index
   return %res0 : !ptr.ptr<#ptr.generic_space>
 }
+
+/// Tests the the `from_ptr` folder.
+// CHECK-LABEL: @test_from_ptr_0
+// CHECK-SAME: (%[[MEM_REF:.*]]: memref<f32, #ptr.generic_space>)
+func.func @test_from_ptr_0(%mr: memref<f32, #ptr.generic_space>) -> memref<f32, #ptr.generic_space> {
+  // CHECK-NOT: ptr.to_ptr
+  // CHECK-NOT: ptr.get_metadata
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK: return %[[MEM_REF]]
+  %ptr = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mda = ptr.get_metadata %mr : memref<f32, #ptr.generic_space>
+  %res = ptr.from_ptr %ptr metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  return %res : memref<f32, #ptr.generic_space>
+}
+
+/// Check the op doesn't fold because folding a ptr-type with metadata requires knowing the origin of the metadata.
+// CHECK-LABEL: @test_from_ptr_1
+// CHECK-SAME: (%[[MEM_REF:.*]]: memref<f32, #ptr.generic_space>)
+func.func @test_from_ptr_1(%mr: memref<f32, #ptr.generic_space>) -> memref<f32, #ptr.generic_space> {
+  // CHECK: ptr.to_ptr
+  // CHECK: ptr.from_ptr
+  %ptr = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %res = ptr.from_ptr %ptr : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  return %res : memref<f32, #ptr.generic_space>
+}
+
+/// Check that the ops cannot be folded because the metadata cannot be guaranteed to be the same.
+// CHECK-LABEL: @test_from_ptr_2
+func.func @test_from_ptr_2(%mr: memref<f32, #ptr.generic_space>, %md: !ptr.ptr_metadata<memref<f32, #ptr.generic_space>>) -> memref<f32, #ptr.generic_space> {
+  // CHECK: ptr.to_ptr
+  // CHECK: ptr.from_ptr
+  %ptr = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %res = ptr.from_ptr %ptr metadata %md : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  return %res : memref<f32, #ptr.generic_space>
+}
+
+// Check the folding of `to_ptr -> from_ptr` chains.
+// CHECK-LABEL: @test_from_ptr_3
+// CHECK-SAME: (%[[MEM_REF:.*]]: memref<f32, #ptr.generic_space>)
+func.func @test_from_ptr_3(%mr0: memref<f32, #ptr.generic_space>) -> memref<f32, #ptr.generic_space> {
+  // CHECK-NOT: ptr.to_ptr
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK: return %[[MEM_REF]]
+  %mda = ptr.get_metadata %mr0 : memref<f32, #ptr.generic_space>
+  %ptr0 = ptr.to_ptr %mr0 : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mrf0 = ptr.from_ptr %ptr0 metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %ptr1 = ptr.to_ptr %mrf0 : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mrf1 = ptr.from_ptr %ptr1 metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  return %mrf1 : memref<f32, #ptr.generic_space>
+}
+
+/// Tests the the `to_ptr` folder.
+// CHECK-LABEL: @test_to_ptr_0
+// CHECK-SAME: (%[[PTR:.*]]: !ptr.ptr<#ptr.generic_space>
+func.func @test_to_ptr_0(%ptr: !ptr.ptr<#ptr.generic_space>, %md: !ptr.ptr_metadata<memref<f32, #ptr.generic_space>>) -> !ptr.ptr<#ptr.generic_space> {
+  // CHECK: return %[[PTR]]
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK-NOT: ptr.to_ptr
+  %mrf = ptr.from_ptr %ptr metadata %md : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %res = ptr.to_ptr %mrf : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  return %res : !ptr.ptr<#ptr.generic_space>
+}
+
+// CHECK-LABEL: @test_to_ptr_1
+// CHECK-SAME: (%[[PTR:.*]]: !ptr.ptr<#ptr.generic_space>)
+func.func @test_to_ptr_1(%ptr: !ptr.ptr<#ptr.generic_space>) -> !ptr.ptr<#ptr.generic_space> {
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK-NOT: ptr.to_ptr
+  // CHECK: return %[[PTR]]
+  %mrf = ptr.from_ptr %ptr : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %res = ptr.to_ptr %mrf : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  return %res : !ptr.ptr<#ptr.generic_space>
+}
+
+// Check the folding of `from_ptr -> to_ptr` chains.
+// CHECK-LABEL: @test_to_ptr_2
+// CHECK-SAME: (%[[PTR:.*]]: !ptr.ptr<#ptr.generic_space>
+func.func @test_to_ptr_2(%ptr0: !ptr.ptr<#ptr.generic_space>) -> !ptr.ptr<#ptr.generic_space> {
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK-NOT: ptr.to_ptr
+  // CHECK: return %[[PTR]]
+  %mrf0 = ptr.from_ptr %ptr0 : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %ptr1 = ptr.to_ptr %mrf0 : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mrf1 = ptr.from_ptr %ptr1 : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %ptr2 = ptr.to_ptr %mrf1 : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mrf2 = ptr.from_ptr %ptr2 : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %res = ptr.to_ptr %mrf2 : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  return %res : !ptr.ptr<#ptr.generic_space>
+}
+
+// Check the folding of chains with different metadata.
+// CHECK-LABEL: @test_cast_chain_folding
+// CHECK-SAME: (%[[MEM_REF:.*]]: memref<f32, #ptr.generic_space>
+func.func @test_cast_chain_folding(%mr: memref<f32, #ptr.generic_space>, %md: !ptr.ptr_metadata<memref<f32, #ptr.generic_space>>) -> memref<f32, #ptr.generic_space> {
+  // CHECK-NOT: ptr.to_ptr
+  // CHECK-NOT: ptr.from_ptr
+  // CHECK: return %[[MEM_REF]]
+   %ptr1 = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+   %memrefWithOtherMd = ptr.from_ptr %ptr1 metadata %md : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+   %ptr = ptr.to_ptr %memrefWithOtherMd : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+   %mda = ptr.get_metadata %mr : memref<f32, #ptr.generic_space>
+   // The chain can be folded because: the ptr always has the same value because
+   // `to_ptr` is a loss-less cast and %mda comes from the original memref.
+   %res = ptr.from_ptr %ptr metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+   return %res : memref<f32, #ptr.generic_space>
+}
diff --git a/mlir/test/Dialect/Ptr/invalid.mlir b/mlir/test/Dialect/Ptr/invalid.mlir
new file mode 100644
index 000000000000..19fd715e5bba
--- /dev/null
+++ b/mlir/test/Dialect/Ptr/invalid.mlir
@@ -0,0 +1,16 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s
+
+/// Test `to_ptr` verifiers.
+func.func @invalid_to_ptr(%v: memref<f32, 0>) {
+  // expected-error@+1 {{expected the input and output to have the same memory space}}
+  %r = ptr.to_ptr %v : memref<f32, 0> -> !ptr.ptr<#ptr.generic_space>
+  return
+}
+
+// -----
+
+func.func @invalid_to_ptr(%v: !ptr.ptr<#ptr.generic_space>) {
+  // expected-error@+1 {{the input value cannot be of type `!ptr.ptr`}}
+  %r = ptr.to_ptr %v : !ptr.ptr<#ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  return
+}
diff --git a/mlir/test/Dialect/Ptr/ops.mlir b/mlir/test/Dialect/Ptr/ops.mlir
index d763ea221944..eed3272d98da 100644
--- a/mlir/test/Dialect/Ptr/ops.mlir
+++ b/mlir/test/Dialect/Ptr/ops.mlir
@@ -17,3 +17,13 @@ func.func @ptr_add_type_offset(%ptr: !ptr.ptr<#ptr.generic_space>) -> !ptr.ptr<#
   %res3 = ptr.ptr_add inbounds %ptr, %off : !ptr.ptr<#ptr.generic_space>, index
   return %res : !ptr.ptr<#ptr.generic_space>
 }
+
+/// Check cast ops assembly.
+// CHECK-LABEL: @cast_ops
+func.func @cast_ops(%mr: memref<f32, #ptr.generic_space>) -> memref<f32, #ptr.generic_space> {
+  %ptr = ptr.to_ptr %mr : memref<f32, #ptr.generic_space> -> !ptr.ptr<#ptr.generic_space>
+  %mda = ptr.get_metadata %mr : memref<f32, #ptr.generic_space>
+  %res = ptr.from_ptr %ptr metadata %mda : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  %mr0 = ptr.from_ptr %ptr : !ptr.ptr<#ptr.generic_space> -> memref<f32, #ptr.generic_space>
+  return %res : memref<f32, #ptr.generic_space>
+}
-- 
cgit v1.2.3


From c103bbc836691d637c249ca19d38bdb0034437c6 Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Fri, 20 Jun 2025 11:37:27 -0700
Subject: [LV] Consider whether vscale is a known power of two for iteration
 check (#144963)

Going mostly by the comment here - but it says "vscale is not
necessarily a power-of-2". Both in tree targets have vscale as a power
of two, and we have an existing TTI hook for that.
---
 llvm/lib/Transforms/Vectorize/LoopVectorize.cpp    |   2 +-
 .../LoopVectorize/AArch64/tail-folding-styles.ll   |  18 +---
 .../LoopVectorize/RISCV/inloop-reduction.ll        |  16 +--
 .../RISCV/type-info-cache-evl-crash.ll             |   6 +-
 ...ectorize-force-tail-with-evl-call-intrinsics.ll |  61 ++---------
 ...ectorize-force-tail-with-evl-cast-intrinsics.ll |  70 ++-----------
 ...vectorize-force-tail-with-evl-cond-reduction.ll |  24 +----
 ...e-force-tail-with-evl-fixed-order-recurrence.ll |  18 +---
 ...ctorize-force-tail-with-evl-inloop-reduction.ll | 112 ++++++---------------
 .../vectorize-force-tail-with-evl-interleave.ll    |   8 +-
 ...orize-force-tail-with-evl-intermediate-store.ll |  12 +--
 .../RISCV/vectorize-force-tail-with-evl-iv32.ll    |   8 +-
 ...torize-force-tail-with-evl-known-no-overflow.ll |  12 +--
 ...ctorize-force-tail-with-evl-masked-loadstore.ll |   8 +-
 ...torize-force-tail-with-evl-ordered-reduction.ll |   8 +-
 .../vectorize-force-tail-with-evl-reduction.ll     | 112 ++++++---------------
 .../vectorize-force-tail-with-evl-uniform-store.ll |   6 +-
 .../LoopVectorize/RISCV/vectorize-vp-intrinsics.ll |   8 +-
 .../RISCV/vplan-vp-select-intrinsics.ll            |  14 +--
 .../LoopVectorize/scalable-predication.ll          |  65 ++++++++++++
 20 files changed, 178 insertions(+), 410 deletions(-)
 create mode 100644 llvm/test/Transforms/LoopVectorize/scalable-predication.ll

diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index ca1e4cbc6b29..f28c2ce0acc9 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2429,7 +2429,7 @@ Value *InnerLoopVectorizer::createIterationCountCheck(ElementCount VF,
       // check is known to be true, or known to be false.
       CheckMinIters = Builder.CreateICmp(P, Count, Step, "min.iters.check");
     } // else step known to be < trip count, use CheckMinIters preset to false.
-  } else if (VF.isScalable() &&
+  } else if (VF.isScalable() && !TTI->isVScaleKnownToBeAPowerOfTwo() &&
              !isIndvarOverflowCheckKnownFalse(Cost, VF, UF) &&
              Style != TailFoldingStyle::DataAndControlFlowWithoutRuntimeCheck) {
     // vscale is not necessarily a power-of-2, which means we cannot guarantee
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
index 1cdc290757dd..0326fe523e67 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll
@@ -54,11 +54,7 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA-LABEL: @simple_memset_tailfold(
 ; DATA-NEXT:  entry:
 ; DATA-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
-; DATA-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
-; DATA-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; DATA-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DATA-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; DATA-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA:       vector.ph:
 ; DATA-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; DATA-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -98,11 +94,7 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_NO_LANEMASK-LABEL: @simple_memset_tailfold(
 ; DATA_NO_LANEMASK-NEXT:  entry:
 ; DATA_NO_LANEMASK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
-; DATA_NO_LANEMASK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
-; DATA_NO_LANEMASK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_NO_LANEMASK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; DATA_NO_LANEMASK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DATA_NO_LANEMASK-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; DATA_NO_LANEMASK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA_NO_LANEMASK:       vector.ph:
 ; DATA_NO_LANEMASK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; DATA_NO_LANEMASK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -150,11 +142,7 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features
 ; DATA_AND_CONTROL-LABEL: @simple_memset_tailfold(
 ; DATA_AND_CONTROL-NEXT:  entry:
 ; DATA_AND_CONTROL-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[N:%.*]], i64 1)
-; DATA_AND_CONTROL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
-; DATA_AND_CONTROL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; DATA_AND_CONTROL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; DATA_AND_CONTROL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; DATA_AND_CONTROL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; DATA_AND_CONTROL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; DATA_AND_CONTROL:       vector.ph:
 ; DATA_AND_CONTROL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; DATA_AND_CONTROL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
index a8e6ef7ebfec..c063e6ff3f48 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll
@@ -344,16 +344,12 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ;
 ; IF-EVL-OUTLOOP-LABEL: @smin(
 ; IF-EVL-OUTLOOP-NEXT:  entry:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-OUTLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-OUTLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -401,16 +397,12 @@ define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ;
 ; IF-EVL-INLOOP-LABEL: @smin(
 ; IF-EVL-INLOOP-NEXT:  entry:
-; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-INLOOP-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-INLOOP-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-INLOOP-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
index a256e92c823e..f059730a245b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/type-info-cache-evl-crash.ll
@@ -10,11 +10,7 @@ define void @type_info_cache_clobber(ptr %dstv, ptr %src, i64 %wide.trip.count)
 ; CHECK-SAME: ptr [[DSTV:%.*]], ptr [[SRC:%.*]], i64 [[WIDE_TRIP_COUNT:%.*]]) #[[ATTR0:[0-9]+]] {
 ; CHECK-NEXT:  [[ENTRY:.*]]:
 ; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[WIDE_TRIP_COUNT]], 1
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 8
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; CHECK:       [[VECTOR_MEMCHECK]]:
 ; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DSTV]], i64 1
 ; CHECK-NEXT:    [[TMP5:%.*]] = add i64 [[WIDE_TRIP_COUNT]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
index 325f3fd2b968..7a5415a61fc7 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-call-intrinsics.ll
@@ -16,12 +16,7 @@ define void @vp_smax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -130,12 +125,7 @@ define void @vp_smin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -244,12 +234,7 @@ define void @vp_umax(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -358,12 +343,7 @@ define void @vp_umin(ptr %a, ptr %b, ptr %c, i64 %N) {
 ; IF-EVL-NEXT:    [[C3:%.*]] = ptrtoint ptr [[C]] to i64
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -472,11 +452,7 @@ define void @vp_ctlz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -571,11 +547,7 @@ define void @vp_cttz(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -670,12 +642,7 @@ define void @vp_lrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -778,12 +745,7 @@ define void @vp_llrint(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 9, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP22:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP22]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -886,12 +848,7 @@ define void @vp_abs(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 8, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP19:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP19]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
index 107ca54c002f..38ed3f24ab23 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cast-intrinsics.ll
@@ -13,12 +13,7 @@ define void @vp_sext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_sext(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
 ; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
@@ -112,12 +107,7 @@ define void @vp_zext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_zext(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
 ; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
@@ -211,12 +201,7 @@ define void @vp_trunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_trunc(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 18, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 2
 ; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
@@ -310,12 +295,7 @@ define void @vp_fpext(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_fpext(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 3
 ; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
@@ -409,12 +389,7 @@ define void @vp_fptrunc(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-LABEL: define void @vp_fptrunc(
 ; IF-EVL-SAME: ptr [[A:%.*]], ptr [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 14, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 2
 ; IF-EVL-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]]
@@ -510,12 +485,7 @@ define void @vp_sitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
@@ -609,12 +579,7 @@ define void @vp_uitofp(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
@@ -708,12 +673,7 @@ define void @vp_fptosi(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
@@ -807,12 +767,7 @@ define void @vp_fptoui(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 10, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 4
@@ -906,12 +861,7 @@ define void @vp_inttoptr(ptr %a, ptr %b, i64 %N) {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
 ; IF-EVL-NEXT:    [[B2:%.*]] = ptrtoint ptr [[B]] to i64
 ; IF-EVL-NEXT:    [[A1:%.*]] = ptrtoint ptr [[A]] to i64
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; IF-EVL-NEXT:    [[TMP3:%.*]] = call i64 @llvm.umax.i64(i64 13, i64 [[TMP2]])
-; IF-EVL-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP3]]
-; IF-EVL-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]]
 ; IF-EVL:       [[VECTOR_MEMCHECK]]:
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
index 8faec471cf5a..e5364136135d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-cond-reduction.ll
@@ -25,11 +25,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @cond_add(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -84,11 +80,7 @@ define i32 @cond_add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-LABEL: define i32 @cond_add(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
-; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
@@ -262,11 +254,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-OUTLOOP-LABEL: define i32 @cond_add_pred(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-OUTLOOP:       vector.ph:
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -335,11 +323,7 @@ define i32 @cond_add_pred(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-INLOOP-LABEL: define i32 @cond_add_pred(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]]) #[[ATTR0]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
-; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL-INLOOP:       vector.ph:
 ; IF-EVL-INLOOP-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-INLOOP-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
index f651f2295b3b..172335d0e447 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-fixed-order-recurrence.ll
@@ -15,11 +15,7 @@ define void @first_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @first_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -156,11 +152,7 @@ define void @second_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @second_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
@@ -318,11 +310,7 @@ define void @third_order_recurrence(ptr noalias %A, ptr noalias %B, i64 %TC) {
 ; IF-EVL-LABEL: define void @third_order_recurrence(
 ; IF-EVL-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i64 [[TC:%.*]]) #[[ATTR0]] {
 ; IF-EVL-NEXT:  [[ENTRY:.*]]:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; IF-EVL-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; IF-EVL:       [[VECTOR_PH]]:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
index c8d52efde4ac..87ac697bf202 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-inloop-reduction.ll
@@ -14,16 +14,12 @@
 define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @add(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -239,16 +235,12 @@ for.end:
 define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @or(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -354,16 +346,12 @@ for.end:
 define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @and(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -469,16 +457,12 @@ for.end:
 define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @xor(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -584,16 +568,12 @@ for.end:
 define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @smin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -702,16 +682,12 @@ for.end:
 define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @smax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -820,16 +796,12 @@ for.end:
 define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @umin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -938,16 +910,12 @@ for.end:
 define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @umax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1056,16 +1024,12 @@ for.end:
 define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fadd(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1281,16 +1245,12 @@ for.end:
 define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-LABEL: @fmin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1401,16 +1361,12 @@ for.end:
 define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-LABEL: @fmax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1745,16 +1701,12 @@ for.end:
 define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmuladd(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1874,16 +1826,12 @@ for.end:
 define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-LABEL: @anyof_icmp(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1998,16 +1946,12 @@ for.end:
 define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-LABEL: @anyof_fcmp(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
index 38ef88457c6c..fe6a693e8381 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-interleave.ll
@@ -13,16 +13,12 @@
 define void @interleave(ptr noalias %a, ptr noalias %b, i64 %N) {
 ; IF-EVL-LABEL: @interleave(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 8
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 8
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[N]], 1
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
index 6d54b219a346..3e804c0ea219 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-intermediate-store.ll
@@ -25,11 +25,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-OUTLOOP-LABEL: define void @reduction_intermediate_store(
 ; IF-EVL-OUTLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-OUTLOOP-NEXT:  entry:
-; IF-EVL-OUTLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-OUTLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-OUTLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-OUTLOOP-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-OUTLOOP-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; IF-EVL-OUTLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; IF-EVL-OUTLOOP:       vector.memcheck:
 ; IF-EVL-OUTLOOP-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ADDR]], i64 4
 ; IF-EVL-OUTLOOP-NEXT:    [[TMP4:%.*]] = shl i64 [[N]], 2
@@ -89,11 +85,7 @@ define void @reduction_intermediate_store(ptr %a, i64 %n, i32 %start, ptr %addr)
 ; IF-EVL-INLOOP-LABEL: define void @reduction_intermediate_store(
 ; IF-EVL-INLOOP-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i32 [[START:%.*]], ptr [[ADDR:%.*]]) #[[ATTR0:[0-9]+]] {
 ; IF-EVL-INLOOP-NEXT:  entry:
-; IF-EVL-INLOOP-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N]]
-; IF-EVL-INLOOP-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-INLOOP-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-INLOOP-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-INLOOP-NEXT:    br i1 [[TMP4]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
+; IF-EVL-INLOOP-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
 ; IF-EVL-INLOOP:       vector.memcheck:
 ; IF-EVL-INLOOP-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[ADDR]], i64 4
 ; IF-EVL-INLOOP-NEXT:    [[TMP5:%.*]] = shl i64 [[N]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
index a05aabc063b1..3d91738d7a0d 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-iv32.ll
@@ -12,16 +12,12 @@
 define void @iv32(ptr noalias %a, ptr noalias %b, i32 %N) {
 ; IF-EVL-LABEL: @iv32(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP19:%.*]] = sub i32 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i32 @llvm.vscale.i32()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i32 [[TMP19]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[ENTRY:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i32 @llvm.vscale.i32()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i32 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i32 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N:%.*]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i32 @llvm.vscale.i32()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
index dcd15087ce3d..258b7ce0e240 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-known-no-overflow.ll
@@ -15,11 +15,7 @@ define void @trip_count_max_1024(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    br i1 [[GUARD]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
 ; CHECK:       [[LOOP_PREHEADER]]:
 ; CHECK-NEXT:    [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TC]], i64 1)
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[UMAX]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
@@ -90,11 +86,7 @@ define void @overflow_at_0(ptr %p, i64 %tc) vscale_range(2, 1024) {
 ; CHECK-NEXT:    [[GUARD:%.*]] = icmp ugt i64 [[TC]], 1024
 ; CHECK-NEXT:    br i1 [[GUARD]], label %[[EXIT:.*]], label %[[LOOP_PREHEADER:.*]]
 ; CHECK:       [[LOOP_PREHEADER]]:
-; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[TC]]
-; CHECK-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 2
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; CHECK-NEXT:    br i1 [[TMP3]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
index 7609182c7690..63acbdfcaf2b 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-masked-loadstore.ll
@@ -12,16 +12,12 @@
 define void @masked_loadstore(ptr noalias %a, ptr noalias %b, i64 %n) {
 ; IF-EVL-LABEL: @masked_loadstore(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
index bc28918dac68..a97c4b303f9f 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-ordered-reduction.ll
@@ -14,16 +14,12 @@
 define float @fadd(ptr noalias nocapture readonly %a, i64 %n) {
 ; IF-EVL-LABEL: @fadd(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
index 53e0b2f45aaa..83deebe42d38 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-reduction.ll
@@ -12,16 +12,12 @@
 define i32 @add(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @add(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -241,16 +237,12 @@ for.end:
 define i32 @or(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @or(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -359,16 +351,12 @@ for.end:
 define i32 @and(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @and(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -477,16 +465,12 @@ for.end:
 define i32 @xor(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @xor(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -595,16 +579,12 @@ for.end:
 define i32 @smin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @smin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -720,16 +700,12 @@ for.end:
 define i32 @smax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @smax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -845,16 +821,12 @@ for.end:
 define i32 @umin(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @umin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -970,16 +942,12 @@ for.end:
 define i32 @umax(ptr %a, i64 %n, i32 %start) {
 ; IF-EVL-LABEL: @umax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1095,16 +1063,12 @@ for.end:
 define float @fadd(ptr %a, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fadd(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1324,16 +1288,12 @@ for.end:
 define float @fmin(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-LABEL: @fmin(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1449,16 +1409,12 @@ for.end:
 define float @fmax(ptr %a, i64 %n, float %start) #0 {
 ; IF-EVL-LABEL: @fmax(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1798,16 +1754,12 @@ for.end:
 define float @fmuladd(ptr %a, ptr %b, i64 %n, float %start) {
 ; IF-EVL-LABEL: @fmuladd(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -1928,16 +1880,12 @@ for.end:
 define i32 @anyof_icmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-LABEL: @anyof_icmp(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
@@ -2052,16 +2000,12 @@ for.end:
 define i32 @anyof_fcmp(ptr %a, i64 %n, i32 %start, i32 %inv) {
 ; IF-EVL-LABEL: @anyof_fcmp(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP6:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP6]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP6]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP7:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
index 81f52627b179..5c94ce180578 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-force-tail-with-evl-uniform-store.ll
@@ -13,11 +13,7 @@ define void @lshift_significand(i32 %n, ptr nocapture writeonly %dst) {
 ; CHECK-NEXT:    [[CMP1_PEEL:%.*]] = icmp eq i32 [[N]], 0
 ; CHECK-NEXT:    [[SPEC_SELECT:%.*]] = select i1 [[CMP1_PEEL]], i64 2, i64 0
 ; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 3, [[SPEC_SELECT]]
-; CHECK-NEXT:    [[TMP1:%.*]] = sub i64 -1, [[TMP0]]
-; CHECK-NEXT:    [[TMP2:%.*]] = call i64 @llvm.vscale.i64()
-; CHECK-NEXT:    [[TMP3:%.*]] = mul nuw i64 [[TMP2]], 2
-; CHECK-NEXT:    [[TMP4:%.*]] = icmp ult i64 [[TMP1]], [[TMP3]]
-; CHECK-NEXT:    br i1 [[TMP4]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
+; CHECK-NEXT:    br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]]
 ; CHECK:       [[VECTOR_PH]]:
 ; CHECK-NEXT:    [[TMP5:%.*]] = call i64 @llvm.vscale.i64()
 ; CHECK-NEXT:    [[TMP6:%.*]] = mul nuw i64 [[TMP5]], 2
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
index f8f397212e0e..7b91f10fa6a0 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vectorize-vp-intrinsics.ll
@@ -12,16 +12,12 @@
 define void @foo(ptr noalias %a, ptr noalias %b, ptr noalias %c, i64 %N) {
 ; IF-EVL-LABEL: @foo(
 ; IF-EVL-NEXT:  entry:
-; IF-EVL-NEXT:    [[TMP0:%.*]] = sub i64 -1, [[N:%.*]]
-; IF-EVL-NEXT:    [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
-; IF-EVL-NEXT:    [[TMP2:%.*]] = mul nuw i64 [[TMP1]], 4
-; IF-EVL-NEXT:    [[TMP3:%.*]] = icmp ult i64 [[TMP0]], [[TMP2]]
-; IF-EVL-NEXT:    br i1 [[TMP3]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; IF-EVL-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
 ; IF-EVL:       vector.ph:
 ; IF-EVL-NEXT:    [[TMP4:%.*]] = call i64 @llvm.vscale.i64()
 ; IF-EVL-NEXT:    [[TMP5:%.*]] = mul nuw i64 [[TMP4]], 4
 ; IF-EVL-NEXT:    [[TMP8:%.*]] = sub i64 [[TMP5]], 1
-; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP8]]
+; IF-EVL-NEXT:    [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP8]]
 ; IF-EVL-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]]
 ; IF-EVL-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
 ; IF-EVL-NEXT:    [[TMP9:%.*]] = call i64 @llvm.vscale.i64()
diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
index 69d4f96f7ba2..543c706250da 100644
--- a/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
+++ b/llvm/test/Transforms/LoopVectorize/RISCV/vplan-vp-select-intrinsics.ll
@@ -20,14 +20,14 @@
  ; IF-EVL-NEXT: Successor(s): ir-bb<scalar.ph>, ir-bb<vector.ph>
 
  ; IF-EVL:      ir-bb<vector.ph>:
- ; IF-EVL-NEXT:   IR   %4 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %5 = mul nuw i64 %4, 4
- ; IF-EVL-NEXT:   IR   %6 = sub i64 %5, 1
- ; IF-EVL-NEXT:   IR   %n.rnd.up = add i64 %N, %6
- ; IF-EVL-NEXT:   IR   %n.mod.vf = urem i64 %n.rnd.up, %5
+ ; IF-EVL-NEXT:   IR   %0 = call i64 @llvm.vscale.i64()
+ ; IF-EVL-NEXT:   IR   %1 = mul nuw i64 %0, 4
+ ; IF-EVL-NEXT:   IR   %2 = sub i64 %1, 1
+ ; IF-EVL-NEXT:   IR   %n.rnd.up = add i64 %N, %2
+ ; IF-EVL-NEXT:   IR   %n.mod.vf = urem i64 %n.rnd.up, %1
  ; IF-EVL-NEXT:   IR   %n.vec = sub i64 %n.rnd.up, %n.mod.vf
- ; IF-EVL-NEXT:   IR   %7 = call i64 @llvm.vscale.i64()
- ; IF-EVL-NEXT:   IR   %8 = mul nuw i64 %7, 4
+ ; IF-EVL-NEXT:   IR   %3 = call i64 @llvm.vscale.i64()
+ ; IF-EVL-NEXT:   IR   %4 = mul nuw i64 %3, 4
  ; IF-EVL-NEXT: Successor(s): vector.body
  ; IF-EVL-EMPTY:
  ; IF-EVL-NEXT: vector.body:
diff --git a/llvm/test/Transforms/LoopVectorize/scalable-predication.ll b/llvm/test/Transforms/LoopVectorize/scalable-predication.ll
new file mode 100644
index 000000000000..8e272debb299
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/scalable-predication.ll
@@ -0,0 +1,65 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -passes=loop-vectorize -force-tail-folding-style=data -prefer-predicate-over-epilogue=predicate-dont-vectorize -force-target-supports-scalable-vectors -S < %s | FileCheck %s
+
+; vscale is not guaranteed to be a power of two, so this test (which
+; deliberately doesn't correspond to an in-tree backend since those
+; *do* have vscale as power-of-two) exercises the code required for the
+; minimum iteration check in the non-power-of-two case.
+define void @foo(i32 %val, ptr dereferenceable(1024) %ptr) {
+; CHECK-LABEL: @foo(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i64 [[TMP6]], 4
+; CHECK-NEXT:    [[TMP8:%.*]] = icmp ult i64 -257, [[TMP7]]
+; CHECK-NEXT:    br i1 [[TMP8]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i64 [[TMP0]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i64 [[TMP1]], 1
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 256, [[TMP2]]
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]]
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[TMP3:%.*]] = call i64 @llvm.vscale.i64()
+; CHECK-NEXT:    [[TMP4:%.*]] = mul nuw i64 [[TMP3]], 4
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT2:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[ACTIVE_LANE_MASK:%.*]] = call <vscale x 4 x i1> @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 256)
+; CHECK-NEXT:    [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP4]]
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    br label [[WHILE_END_LOOPEXIT:%.*]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    br label [[WHILE_BODY:%.*]]
+; CHECK:       while.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[WHILE_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ]
+; CHECK-NEXT:    [[GEP:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[INDEX]]
+; CHECK-NEXT:    [[LD1:%.*]] = load i32, ptr [[GEP]], align 4
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nsw i64 [[INDEX]], 1
+; CHECK-NEXT:    [[CMP10:%.*]] = icmp ult i64 [[INDEX_NEXT]], 256
+; CHECK-NEXT:    br i1 [[CMP10]], label [[WHILE_BODY]], label [[WHILE_END_LOOPEXIT]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       while.end.loopexit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  br label %while.body
+
+while.body:                                       ; preds = %while.body, %entry
+  %index = phi i64 [ %index.next, %while.body ], [ 0, %entry ]
+  %gep = getelementptr i32, ptr %ptr, i64 %index
+  %ld1 = load i32, ptr %gep, align 4
+  %index.next = add nsw i64 %index, 1
+  %cmp10 = icmp ult i64 %index.next, 256
+  br i1 %cmp10, label %while.body, label %while.end.loopexit, !llvm.loop !0
+
+while.end.loopexit:                               ; preds = %while.body
+  ret void
+}
+
+!0 = distinct !{!0, !1, !2, !3, !4}
+!1 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
+!2 = !{!"llvm.loop.vectorize.scalable.enable", i1 true}
+!3 = !{!"llvm.loop.interleave.count", i32 1}
+!4 = !{!"llvm.loop.vectorize.width", i32 4}
-- 
cgit v1.2.3


From 8d2eea96b391c5346543eceae5c8d24efe4f4497 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:06:55 -0700
Subject: [AMDGPU] gfx1250 SOPP MC tests. NFC. (#145082)

---
 llvm/test/MC/AMDGPU/gfx12_asm_sopp.s               | 71 +++++++++++-----------
 .../MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt     | 25 ++++----
 2 files changed, 49 insertions(+), 47 deletions(-)

diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
index 97b6e3ba22b0..a58d68cb30cb 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
@@ -1,4 +1,5 @@
-// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefix=GFX12 %s
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1200 %s | FileCheck --check-prefixes=GFX12,GFX1200 %s
+// RUN: not llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1250 %s | FileCheck --check-prefix=GFX12 %s
 
 s_wait_loadcnt 0x1234
 // GFX12: encoding: [0x34,0x12,0xc0,0xbf]
@@ -13,22 +14,22 @@ s_wait_storecnt 0xc1d1
 // GFX12: encoding: [0xd1,0xc1,0xc1,0xbf]
 
 s_wait_samplecnt 0x1234
-// GFX12: encoding: [0x34,0x12,0xc2,0xbf]
+// GFX1200: encoding: [0x34,0x12,0xc2,0xbf]
 
 s_wait_samplecnt 0xc1d1
-// GFX12: encoding: [0xd1,0xc1,0xc2,0xbf]
+// GFX1200: encoding: [0xd1,0xc1,0xc2,0xbf]
 
 s_wait_bvhcnt 0x1234
-// GFX12: encoding: [0x34,0x12,0xc3,0xbf]
+// GFX1200: encoding: [0x34,0x12,0xc3,0xbf]
 
 s_wait_bvhcnt 0xc1d1
-// GFX12: encoding: [0xd1,0xc1,0xc3,0xbf]
+// GFX1200: encoding: [0xd1,0xc1,0xc3,0xbf]
 
 s_wait_expcnt 0x1234
-// GFX12: encoding: [0x34,0x12,0xc4,0xbf]
+// GFX1200: encoding: [0x34,0x12,0xc4,0xbf]
 
 s_wait_expcnt 0xc1d1
-// GFX12: encoding: [0xd1,0xc1,0xc4,0xbf]
+// GFX1200: encoding: [0xd1,0xc1,0xc4,0xbf]
 
 s_wait_dscnt 0x1234
 // GFX12: encoding: [0x34,0x12,0xc6,0xbf]
@@ -80,73 +81,73 @@ s_barrier_wait 1
 //===----------------------------------------------------------------------===//
 
 s_waitcnt 0
-// GFX12: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
 
 s_waitcnt 0x1234
-// GFX12: s_waitcnt vmcnt(4) expcnt(4) lgkmcnt(35) ; encoding: [0x34,0x12,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(4) expcnt(4) lgkmcnt(35) ; encoding: [0x34,0x12,0x89,0xbf]
 
 s_waitcnt vmcnt(0) & expcnt(0) & lgkmcnt(0)
-// GFX12: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
 
 s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-// GFX12: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
 
 s_waitcnt vmcnt(0), expcnt(0), lgkmcnt(0)
-// GFX12: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; encoding: [0x00,0x00,0x89,0xbf]
 
 s_waitcnt vmcnt(1)
-// GFX12: s_waitcnt vmcnt(1) ; encoding: [0xf7,0x07,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(1) ; encoding: [0xf7,0x07,0x89,0xbf]
 
 s_waitcnt vmcnt(9)
-// GFX12: s_waitcnt vmcnt(9) ; encoding: [0xf7,0x27,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(9) ; encoding: [0xf7,0x27,0x89,0xbf]
 
 s_waitcnt expcnt(2)
-// GFX12: s_waitcnt expcnt(2) ; encoding: [0xf2,0xff,0x89,0xbf]
+// GFX1200: s_waitcnt expcnt(2) ; encoding: [0xf2,0xff,0x89,0xbf]
 
 s_waitcnt lgkmcnt(3)
-// GFX12: s_waitcnt lgkmcnt(3) ; encoding: [0x37,0xfc,0x89,0xbf]
+// GFX1200: s_waitcnt lgkmcnt(3) ; encoding: [0x37,0xfc,0x89,0xbf]
 
 s_waitcnt lgkmcnt(9)
-// GFX12: s_waitcnt lgkmcnt(9) ; encoding: [0x97,0xfc,0x89,0xbf]
+// GFX1200: s_waitcnt lgkmcnt(9) ; encoding: [0x97,0xfc,0x89,0xbf]
 
 s_waitcnt vmcnt(0), expcnt(0)
-// GFX12: s_waitcnt vmcnt(0) expcnt(0) ; encoding: [0xf0,0x03,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(0) expcnt(0) ; encoding: [0xf0,0x03,0x89,0xbf]
 
 s_waitcnt vmcnt(15)
-// GFX12: s_waitcnt vmcnt(15) ; encoding: [0xf7,0x3f,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(15) ; encoding: [0xf7,0x3f,0x89,0xbf]
 
 s_waitcnt vmcnt(15) expcnt(6)
-// GFX12: s_waitcnt vmcnt(15) expcnt(6) ; encoding: [0xf6,0x3f,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(15) expcnt(6) ; encoding: [0xf6,0x3f,0x89,0xbf]
 
 s_waitcnt vmcnt(15) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(15) lgkmcnt(14) ; encoding: [0xe7,0x3c,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(15) lgkmcnt(14) ; encoding: [0xe7,0x3c,0x89,0xbf]
 
 s_waitcnt vmcnt(15) expcnt(6) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(15) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0x3c,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(15) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0x3c,0x89,0xbf]
 
 s_waitcnt vmcnt(31)
-// GFX12: s_waitcnt vmcnt(31) ; encoding: [0xf7,0x7f,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(31) ; encoding: [0xf7,0x7f,0x89,0xbf]
 
 s_waitcnt vmcnt(31) expcnt(6)
-// GFX12: s_waitcnt vmcnt(31) expcnt(6) ; encoding: [0xf6,0x7f,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(31) expcnt(6) ; encoding: [0xf6,0x7f,0x89,0xbf]
 
 s_waitcnt vmcnt(31) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(31) lgkmcnt(14) ; encoding: [0xe7,0x7c,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(31) lgkmcnt(14) ; encoding: [0xe7,0x7c,0x89,0xbf]
 
 s_waitcnt vmcnt(31) expcnt(6) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(31) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0x7c,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(31) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0x7c,0x89,0xbf]
 
 s_waitcnt vmcnt(62)
-// GFX12: s_waitcnt vmcnt(62) ; encoding: [0xf7,0xfb,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(62) ; encoding: [0xf7,0xfb,0x89,0xbf]
 
 s_waitcnt vmcnt(62) expcnt(6)
-// GFX12: s_waitcnt vmcnt(62) expcnt(6) ; encoding: [0xf6,0xfb,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(62) expcnt(6) ; encoding: [0xf6,0xfb,0x89,0xbf]
 
 s_waitcnt vmcnt(62) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(62) lgkmcnt(14) ; encoding: [0xe7,0xf8,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(62) lgkmcnt(14) ; encoding: [0xe7,0xf8,0x89,0xbf]
 
 s_waitcnt vmcnt(62) expcnt(6) lgkmcnt(14)
-// GFX12: s_waitcnt vmcnt(62) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0xf8,0x89,0xbf]
+// GFX1200: s_waitcnt vmcnt(62) expcnt(6) lgkmcnt(14) ; encoding: [0xe6,0xf8,0x89,0xbf]
 
 //===----------------------------------------------------------------------===//
 // s_sendmsg
@@ -259,13 +260,13 @@ s_cbranch_execnz 0x1234
 // GFX12: s_cbranch_execnz 4660 ; encoding: [0x34,0x12,0xa6,0xbf]
 
 s_setkill 0x0
-// GFX12: s_setkill 0 ; encoding: [0x00,0x00,0x81,0xbf]
+// GFX1200: s_setkill 0 ; encoding: [0x00,0x00,0x81,0xbf]
 
 s_setkill 0x1234
-// GFX12: s_setkill 0x1234 ; encoding: [0x34,0x12,0x81,0xbf]
+// GFX1200: s_setkill 0x1234 ; encoding: [0x34,0x12,0x81,0xbf]
 
 s_setkill 0xc1d1
-// GFX12: s_setkill 0xc1d1 ; encoding: [0xd1,0xc1,0x81,0xbf]
+// GFX1200: s_setkill 0xc1d1 ; encoding: [0xd1,0xc1,0x81,0xbf]
 
 s_sethalt 0x0
 // GFX12: s_sethalt 0 ; encoding: [0x00,0x00,0x82,0xbf]
@@ -379,7 +380,7 @@ s_ttracedata_imm 0xc1d1
 // GFX12: s_ttracedata_imm 0xc1d1 ; encoding: [0xd1,0xc1,0xbb,0xbf]
 
 s_wait_event 0x3141
-// GFX12: s_wait_event 0x3141 ; encoding: [0x41,0x31,0x8b,0xbf]
+// GFX1200: s_wait_event 0x3141 ; encoding: [0x41,0x31,0x8b,0xbf]
 
 s_wait_event 0xc1d1
-// GFX12: s_wait_event 0xc1d1 ; encoding: [0xd1,0xc1,0x8b,0xbf]
+// GFX1200: s_wait_event 0xc1d1 ; encoding: [0xd1,0xc1,0x8b,0xbf]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
index 266ebf3822d3..f8aef7267862 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
@@ -1,4 +1,5 @@
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX12 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX12,GFX1200 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefixes=GFX12 %s
 
 # GFX12: s_wait_alu depctr_hold_cnt(0) depctr_sa_sdst(0) depctr_va_vdst(0) depctr_va_sdst(0) depctr_va_ssrc(0) depctr_va_vcc(0) depctr_vm_vsrc(0) ; encoding: [0x00,0x00,0x88,0xbf]
 0x00,0x00,0x88,0xbf
@@ -18,22 +19,22 @@
 # GFX12: s_wait_storecnt 0xc1d1                  ; encoding: [0xd1,0xc1,0xc1,0xbf]
 0xd1,0xc1,0xc1,0xbf
 
-# GFX12: s_wait_samplecnt 0x1234                 ; encoding: [0x34,0x12,0xc2,0xbf]
+# GFX1200: s_wait_samplecnt 0x1234                 ; encoding: [0x34,0x12,0xc2,0xbf]
 0x34,0x12,0xc2,0xbf
 
-# GFX12: s_wait_samplecnt 0xc1d1                 ; encoding: [0xd1,0xc1,0xc2,0xbf]
+# GFX1200: s_wait_samplecnt 0xc1d1                 ; encoding: [0xd1,0xc1,0xc2,0xbf]
 0xd1,0xc1,0xc2,0xbf
 
-# GFX12: s_wait_bvhcnt 0x1234                    ; encoding: [0x34,0x12,0xc3,0xbf]
+# GFX1200: s_wait_bvhcnt 0x1234                    ; encoding: [0x34,0x12,0xc3,0xbf]
 0x34,0x12,0xc3,0xbf
 
-# GFX12: s_wait_bvhcnt 0xc1d1                    ; encoding: [0xd1,0xc1,0xc3,0xbf]
+# GFX1200: s_wait_bvhcnt 0xc1d1                    ; encoding: [0xd1,0xc1,0xc3,0xbf]
 0xd1,0xc1,0xc3,0xbf
 
-# GFX12: s_wait_expcnt 0x1234                    ; encoding: [0x34,0x12,0xc4,0xbf]
+# GFX1200: s_wait_expcnt 0x1234                    ; encoding: [0x34,0x12,0xc4,0xbf]
 0x34,0x12,0xc4,0xbf
 
-# GFX12: s_wait_expcnt 0xc1d1                    ; encoding: [0xd1,0xc1,0xc4,0xbf]
+# GFX1200: s_wait_expcnt 0xc1d1                    ; encoding: [0xd1,0xc1,0xc4,0xbf]
 0xd1,0xc1,0xc4,0xbf
 
 # GFX12: s_wait_dscnt 0x1234                     ; encoding: [0x34,0x12,0xc6,0xbf]
@@ -250,13 +251,13 @@
 # GFX12: s_sethalt 0xc1d1                        ; encoding: [0xd1,0xc1,0x82,0xbf]
 0xd1,0xc1,0x82,0xbf
 
-# GFX12: s_setkill 0                             ; encoding: [0x00,0x00,0x81,0xbf]
+# GFX1200: s_setkill 0                             ; encoding: [0x00,0x00,0x81,0xbf]
 0x00,0x00,0x81,0xbf
 
-# GFX12: s_setkill 0x1234                        ; encoding: [0x34,0x12,0x81,0xbf]
+# GFX1200: s_setkill 0x1234                        ; encoding: [0x34,0x12,0x81,0xbf]
 0x34,0x12,0x81,0xbf
 
-# GFX12: s_setkill 0xc1d1                        ; encoding: [0xd1,0xc1,0x81,0xbf]
+# GFX1200: s_setkill 0xc1d1                        ; encoding: [0xd1,0xc1,0x81,0xbf]
 0xd1,0xc1,0x81,0xbf
 
 # GFX12: s_setprio 0                             ; encoding: [0x00,0x00,0xb5,0xbf]
@@ -304,8 +305,8 @@
 # GFX12: s_wakeup                                ; encoding: [0x00,0x00,0xb4,0xbf]
 0x00,0x00,0xb4,0xbf
 
-# GFX12: s_wait_event 0x3141                     ; encoding: [0x41,0x31,0x8b,0xbf]
+# GFX1200: s_wait_event 0x3141                     ; encoding: [0x41,0x31,0x8b,0xbf]
 0x41,0x31,0x8b,0xbf
 
-# GFX12: s_wait_event 0xc1d1                     ; encoding: [0xd1,0xc1,0x8b,0xbf]
+# GFX1200: s_wait_event 0xc1d1                     ; encoding: [0xd1,0xc1,0x8b,0xbf]
 0xd1,0xc1,0x8b,0xbf
-- 
cgit v1.2.3


From b5d5708128e99f69add50c322bfbed5f4905c23d Mon Sep 17 00:00:00 2001
From: joaosaffran <126493771+joaosaffran@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:12:02 -0700
Subject: [HLSL] Add descriptor table metadata parsing (#142492)

Implements descriptor table parsing from root signature metadata. This
is required to support root signatures in hlsl.
Closes: #[126640](https://github.com/llvm/llvm-project/issues/126640)

---------

Co-authored-by: joaosaffran <joao.saffran@microsoft.com>
---
 llvm/include/llvm/BinaryFormat/DXContainer.h       |   5 +
 .../llvm/BinaryFormat/DXContainerConstants.def     |  10 +
 llvm/lib/Target/DirectX/DXILRootSignature.cpp      | 245 ++++++++++++++++++---
 llvm/lib/Target/DirectX/DXILRootSignature.h        |   1 +
 ...ure-DescriptorTable-AllValidFlagCombinations.ll | 157 +++++++++++++
 ...e-DescriptorTable-AllValidFlagCombinationsV1.ll |  44 ++++
 .../RootSignature-DescriptorTable-Invalid-Flag.ll  |  20 ++
 ...tSignature-DescriptorTable-Invalid-RangeType.ll |  20 ++
 ...nature-DescriptorTable-Invalid-RegisterSpace.ll |  20 ++
 .../ContainerData/RootSignature-DescriptorTable.ll |  48 ++++
 .../ContainerData/RootSignature-Parameters.ll      |  22 +-
 .../RootSignature-RootDescriptor-Invalid-Flags.ll  |   2 +-
 12 files changed, 559 insertions(+), 35 deletions(-)
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinations.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinationsV1.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-Flag.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RangeType.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RegisterSpace.ll
 create mode 100644 llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable.ll

diff --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index 6d625dad5853..56c9e5330867 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_BINARYFORMAT_DXCONTAINER_H
 #define LLVM_BINARYFORMAT_DXCONTAINER_H
 
+#include "llvm/ADT/BitmaskEnum.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Error.h"
@@ -40,6 +41,8 @@ template <typename T> struct EnumEntry;
 
 namespace dxbc {
 
+LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE();
+
 inline Triple::EnvironmentType getShaderStage(uint32_t Kind) {
   assert(Kind <= Triple::Amplification - Triple::Pixel &&
          "Shader kind out of expected range.");
@@ -167,6 +170,8 @@ enum class RootDescriptorFlag : uint32_t {
 #define DESCRIPTOR_RANGE_FLAG(Num, Val) Val = Num,
 enum class DescriptorRangeFlag : uint32_t {
 #include "DXContainerConstants.def"
+
+  LLVM_MARK_AS_BITMASK_ENUM(DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
 };
 
 #define ROOT_PARAMETER(Val, Enum) Enum = Val,
diff --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index 18e79e6fa65a..a281256a44ae 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -99,6 +99,16 @@ DESCRIPTOR_RANGE_FLAG(0x10000, DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS)
 #undef DESCRIPTOR_RANGE_FLAG
 #endif // DESCRIPTOR_RANGE_FLAG
 
+// DESCRIPTOR_RANGE(value, name).
+#ifdef DESCRIPTOR_RANGE
+
+DESCRIPTOR_RANGE(0, SRV)
+DESCRIPTOR_RANGE(1, UAV)
+DESCRIPTOR_RANGE(2, CBV)
+DESCRIPTOR_RANGE(3, Sampler)
+#undef DESCRIPTOR_RANGE
+#endif // DESCRIPTOR_RANGE
+
 #ifdef ROOT_PARAMETER
 
 ROOT_PARAMETER(0, DescriptorTable)
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 88914a31f46e..1f175fd4ecd9 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -174,6 +174,93 @@ static bool parseRootDescriptors(LLVMContext *Ctx,
   return false;
 }
 
+static bool parseDescriptorRange(LLVMContext *Ctx,
+                                 mcdxbc::DescriptorTable &Table,
+                                 MDNode *RangeDescriptorNode) {
+
+  if (RangeDescriptorNode->getNumOperands() != 6)
+    return reportError(Ctx, "Invalid format for Descriptor Range");
+
+  dxbc::RTS0::v2::DescriptorRange Range;
+
+  std::optional<StringRef> ElementText =
+      extractMdStringValue(RangeDescriptorNode, 0);
+
+  if (!ElementText.has_value())
+    return reportError(Ctx, "Descriptor Range, first element is not a string.");
+
+  Range.RangeType =
+      StringSwitch<uint32_t>(*ElementText)
+          .Case("CBV", llvm::to_underlying(dxbc::DescriptorRangeType::CBV))
+          .Case("SRV", llvm::to_underlying(dxbc::DescriptorRangeType::SRV))
+          .Case("UAV", llvm::to_underlying(dxbc::DescriptorRangeType::UAV))
+          .Case("Sampler",
+                llvm::to_underlying(dxbc::DescriptorRangeType::Sampler))
+          .Default(~0U);
+
+  if (Range.RangeType == ~0U)
+    return reportError(Ctx, "Invalid Descriptor Range type: " + *ElementText);
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RangeDescriptorNode, 1))
+    Range.NumDescriptors = *Val;
+  else
+    return reportError(Ctx, "Invalid value for Number of Descriptor in Range");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RangeDescriptorNode, 2))
+    Range.BaseShaderRegister = *Val;
+  else
+    return reportError(Ctx, "Invalid value for BaseShaderRegister");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RangeDescriptorNode, 3))
+    Range.RegisterSpace = *Val;
+  else
+    return reportError(Ctx, "Invalid value for RegisterSpace");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RangeDescriptorNode, 4))
+    Range.OffsetInDescriptorsFromTableStart = *Val;
+  else
+    return reportError(Ctx,
+                       "Invalid value for OffsetInDescriptorsFromTableStart");
+
+  if (std::optional<uint32_t> Val = extractMdIntValue(RangeDescriptorNode, 5))
+    Range.Flags = *Val;
+  else
+    return reportError(Ctx, "Invalid value for Descriptor Range Flags");
+
+  Table.Ranges.push_back(Range);
+  return false;
+}
+
+static bool parseDescriptorTable(LLVMContext *Ctx,
+                                 mcdxbc::RootSignatureDesc &RSD,
+                                 MDNode *DescriptorTableNode) {
+  const unsigned int NumOperands = DescriptorTableNode->getNumOperands();
+  if (NumOperands < 2)
+    return reportError(Ctx, "Invalid format for Descriptor Table");
+
+  dxbc::RTS0::v1::RootParameterHeader Header;
+  if (std::optional<uint32_t> Val = extractMdIntValue(DescriptorTableNode, 1))
+    Header.ShaderVisibility = *Val;
+  else
+    return reportError(Ctx, "Invalid value for ShaderVisibility");
+
+  mcdxbc::DescriptorTable Table;
+  Header.ParameterType =
+      llvm::to_underlying(dxbc::RootParameterType::DescriptorTable);
+
+  for (unsigned int I = 2; I < NumOperands; I++) {
+    MDNode *Element = dyn_cast<MDNode>(DescriptorTableNode->getOperand(I));
+    if (Element == nullptr)
+      return reportError(Ctx, "Missing Root Element Metadata Node.");
+
+    if (parseDescriptorRange(Ctx, Table, Element))
+      return true;
+  }
+
+  RSD.ParametersContainer.addParameter(Header, Table);
+  return false;
+}
+
 static bool parseRootSignatureElement(LLVMContext *Ctx,
                                       mcdxbc::RootSignatureDesc &RSD,
                                       MDNode *Element) {
@@ -188,6 +275,7 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
           .Case("RootCBV", RootSignatureElementKind::CBV)
           .Case("RootSRV", RootSignatureElementKind::SRV)
           .Case("RootUAV", RootSignatureElementKind::UAV)
+          .Case("DescriptorTable", RootSignatureElementKind::DescriptorTable)
           .Default(RootSignatureElementKind::Error);
 
   switch (ElementKind) {
@@ -200,6 +288,8 @@ static bool parseRootSignatureElement(LLVMContext *Ctx,
   case RootSignatureElementKind::SRV:
   case RootSignatureElementKind::UAV:
     return parseRootDescriptors(Ctx, RSD, Element, ElementKind);
+  case RootSignatureElementKind::DescriptorTable:
+    return parseDescriptorTable(Ctx, RSD, Element);
   case RootSignatureElementKind::Error:
     return reportError(Ctx, "Invalid Root Signature Element: " + *ElementText);
   }
@@ -241,6 +331,81 @@ static bool verifyRegisterSpace(uint32_t RegisterSpace) {
 
 static bool verifyDescriptorFlag(uint32_t Flags) { return (Flags & ~0xE) == 0; }
 
+static bool verifyRangeType(uint32_t Type) {
+  switch (Type) {
+  case llvm::to_underlying(dxbc::DescriptorRangeType::CBV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::SRV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::UAV):
+  case llvm::to_underlying(dxbc::DescriptorRangeType::Sampler):
+    return true;
+  };
+
+  return false;
+}
+
+static bool verifyDescriptorRangeFlag(uint32_t Version, uint32_t Type,
+                                      uint32_t FlagsVal) {
+  using FlagT = dxbc::DescriptorRangeFlag;
+  FlagT Flags = FlagT(FlagsVal);
+
+  const bool IsSampler =
+      (Type == llvm::to_underlying(dxbc::DescriptorRangeType::Sampler));
+
+  if (Version == 1) {
+    // Since the metadata is unversioned, we expect to explicitly see the values
+    // that map to the version 1 behaviour here.
+    if (IsSampler)
+      return Flags == FlagT::DESCRIPTORS_VOLATILE;
+    return Flags == (FlagT::DATA_VOLATILE | FlagT::DESCRIPTORS_VOLATILE);
+  }
+
+  // The data-specific flags are mutually exclusive.
+  FlagT DataFlags = FlagT::DATA_VOLATILE | FlagT::DATA_STATIC |
+                    FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+
+  if (popcount(llvm::to_underlying(Flags & DataFlags)) > 1)
+    return false;
+
+  // The descriptor-specific flags are mutually exclusive.
+  FlagT DescriptorFlags =
+      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS |
+      FlagT::DESCRIPTORS_VOLATILE;
+  if (popcount(llvm::to_underlying(Flags & DescriptorFlags)) > 1)
+    return false;
+
+  // For volatile descriptors, DATA_STATIC is never valid.
+  if ((Flags & FlagT::DESCRIPTORS_VOLATILE) == FlagT::DESCRIPTORS_VOLATILE) {
+    FlagT Mask = FlagT::DESCRIPTORS_VOLATILE;
+    if (!IsSampler) {
+      Mask |= FlagT::DATA_VOLATILE;
+      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+    }
+    return (Flags & ~Mask) == FlagT::NONE;
+  }
+
+  // For "STATIC_KEEPING_BUFFER_BOUNDS_CHECKS" descriptors,
+  // the other data-specific flags may all be set.
+  if ((Flags & FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) ==
+      FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS) {
+    FlagT Mask = FlagT::DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS;
+    if (!IsSampler) {
+      Mask |= FlagT::DATA_VOLATILE;
+      Mask |= FlagT::DATA_STATIC;
+      Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+    }
+    return (Flags & ~Mask) == FlagT::NONE;
+  }
+
+  // When no descriptor flag is set, any data flag is allowed.
+  FlagT Mask = FlagT::NONE;
+  if (!IsSampler) {
+    Mask |= FlagT::DATA_VOLATILE;
+    Mask |= FlagT::DATA_STATIC;
+    Mask |= FlagT::DATA_STATIC_WHILE_SET_AT_EXECUTE;
+  }
+  return (Flags & ~Mask) == FlagT::NONE;
+}
+
 static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
   if (!verifyVersion(RSD.Version)) {
@@ -275,7 +440,23 @@ static bool validate(LLVMContext *Ctx, const mcdxbc::RootSignatureDesc &RSD) {
 
       if (RSD.Version > 1) {
         if (!verifyDescriptorFlag(Descriptor.Flags))
-          return reportValueError(Ctx, "DescriptorFlag", Descriptor.Flags);
+          return reportValueError(Ctx, "DescriptorRangeFlag", Descriptor.Flags);
+      }
+      break;
+    }
+    case llvm::to_underlying(dxbc::RootParameterType::DescriptorTable): {
+      const mcdxbc::DescriptorTable &Table =
+          RSD.ParametersContainer.getDescriptorTable(Info.Location);
+      for (const dxbc::RTS0::v2::DescriptorRange &Range : Table) {
+        if (!verifyRangeType(Range.RangeType))
+          return reportValueError(Ctx, "RangeType", Range.RangeType);
+
+        if (!verifyRegisterSpace(Range.RegisterSpace))
+          return reportValueError(Ctx, "RegisterSpace", Range.RegisterSpace);
+
+        if (!verifyDescriptorRangeFlag(RSD.Version, Range.RangeType,
+                                       Range.Flags))
+          return reportValueError(Ctx, "DescriptorFlag", Range.Flags);
       }
       break;
     }
@@ -388,43 +569,33 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M,
 
   OS << "Root Signature Definitions"
      << "\n";
-  uint8_t Space = 0;
   for (const Function &F : M) {
     auto It = RSDMap.find(&F);
     if (It == RSDMap.end())
       continue;
     const auto &RS = It->second;
     OS << "Definition for '" << F.getName() << "':\n";
-
     // start root signature header
-    Space++;
-    OS << indent(Space) << "Flags: " << format_hex(RS.Flags, 8) << "\n";
-    OS << indent(Space) << "Version: " << RS.Version << "\n";
-    OS << indent(Space) << "RootParametersOffset: " << RS.RootParameterOffset
-       << "\n";
-    OS << indent(Space) << "NumParameters: " << RS.ParametersContainer.size()
-       << "\n";
-    Space++;
+    OS << "Flags: " << format_hex(RS.Flags, 8) << "\n"
+       << "Version: " << RS.Version << "\n"
+       << "RootParametersOffset: " << RS.RootParameterOffset << "\n"
+       << "NumParameters: " << RS.ParametersContainer.size() << "\n";
     for (size_t I = 0; I < RS.ParametersContainer.size(); I++) {
       const auto &[Type, Loc] =
           RS.ParametersContainer.getTypeAndLocForParameter(I);
       const dxbc::RTS0::v1::RootParameterHeader Header =
           RS.ParametersContainer.getHeader(I);
 
-      OS << indent(Space) << "- Parameter Type: " << Type << "\n";
-      OS << indent(Space + 2)
-         << "Shader Visibility: " << Header.ShaderVisibility << "\n";
+      OS << "- Parameter Type: " << Type << "\n"
+         << "  Shader Visibility: " << Header.ShaderVisibility << "\n";
 
       switch (Type) {
       case llvm::to_underlying(dxbc::RootParameterType::Constants32Bit): {
         const dxbc::RTS0::v1::RootConstants &Constants =
             RS.ParametersContainer.getConstant(Loc);
-        OS << indent(Space + 2) << "Register Space: " << Constants.RegisterSpace
-           << "\n";
-        OS << indent(Space + 2)
-           << "Shader Register: " << Constants.ShaderRegister << "\n";
-        OS << indent(Space + 2)
-           << "Num 32 Bit Values: " << Constants.Num32BitValues << "\n";
+        OS << "  Register Space: " << Constants.RegisterSpace << "\n"
+           << "  Shader Register: " << Constants.ShaderRegister << "\n"
+           << "  Num 32 Bit Values: " << Constants.Num32BitValues << "\n";
         break;
       }
       case llvm::to_underlying(dxbc::RootParameterType::CBV):
@@ -432,23 +603,33 @@ PreservedAnalyses RootSignatureAnalysisPrinter::run(Module &M,
       case llvm::to_underlying(dxbc::RootParameterType::SRV): {
         const dxbc::RTS0::v2::RootDescriptor &Descriptor =
             RS.ParametersContainer.getRootDescriptor(Loc);
-        OS << indent(Space + 2)
-           << "Register Space: " << Descriptor.RegisterSpace << "\n";
-        OS << indent(Space + 2)
-           << "Shader Register: " << Descriptor.ShaderRegister << "\n";
+        OS << "  Register Space: " << Descriptor.RegisterSpace << "\n"
+           << "  Shader Register: " << Descriptor.ShaderRegister << "\n";
         if (RS.Version > 1)
-          OS << indent(Space + 2) << "Flags: " << Descriptor.Flags << "\n";
+          OS << "  Flags: " << Descriptor.Flags << "\n";
+        break;
+      }
+      case llvm::to_underlying(dxbc::RootParameterType::DescriptorTable): {
+        const mcdxbc::DescriptorTable &Table =
+            RS.ParametersContainer.getDescriptorTable(Loc);
+        OS << "  NumRanges: " << Table.Ranges.size() << "\n";
+
+        for (const dxbc::RTS0::v2::DescriptorRange Range : Table) {
+          OS << "  - Range Type: " << Range.RangeType << "\n"
+             << "    Register Space: " << Range.RegisterSpace << "\n"
+             << "    Base Shader Register: " << Range.BaseShaderRegister << "\n"
+             << "    Num Descriptors: " << Range.NumDescriptors << "\n"
+             << "    Offset In Descriptors From Table Start: "
+             << Range.OffsetInDescriptorsFromTableStart << "\n";
+          if (RS.Version > 1)
+            OS << "    Flags: " << Range.Flags << "\n";
+        }
         break;
       }
       }
-      Space--;
     }
-    OS << indent(Space) << "NumStaticSamplers: " << 0 << "\n";
-    OS << indent(Space) << "StaticSamplersOffset: " << RS.StaticSamplersOffset
-       << "\n";
-
-    Space--;
-    // end root signature header
+    OS << "NumStaticSamplers: " << 0 << "\n";
+    OS << "StaticSamplersOffset: " << RS.StaticSamplersOffset << "\n";
   }
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.h b/llvm/lib/Target/DirectX/DXILRootSignature.h
index 3f25551b2b5e..b45cebc15fd3 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.h
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.h
@@ -31,6 +31,7 @@ enum class RootSignatureElementKind {
   SRV = 3,
   UAV = 4,
   CBV = 5,
+  DescriptorTable = 6,
 };
 class RootSignatureAnalysis : public AnalysisInfoMixin<RootSignatureAnalysis> {
   friend AnalysisInfoMixin<RootSignatureAnalysis>;
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinations.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinations.ll
new file mode 100644
index 000000000000..8eb7f90c6b75
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinations.ll
@@ -0,0 +1,157 @@
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !8, !9, !10, !11, !12, !13, !14, !15, !16, !17, !18, !19, !20 }
+
+; typedef enum D3D12_DESCRIPTOR_RANGE_FLAGS {
+;   NONE = 0,
+;   DESCRIPTORS_VOLATILE = 0x1,
+;   DATA_VOLATILE = 0x2,
+;   DATA_STATIC_WHILE_SET_AT_EXECUTE = 0x4,
+;   DATA_STATIC = 0x8,
+;   DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS = 0x10000
+; } ;
+
+;0
+!6 = !{ !"Sampler", i32 1, i32 0, i32 1, i32 -1, i32 0 }
+;DESCRIPTORS_VOLATILE
+!8 = !{ !"Sampler", i32 1, i32 0, i32 3, i32 -1, i32 1 }
+;DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS
+!9 = !{ !"Sampler", i32 1, i32 0, i32 4, i32 -1, i32 65536 }
+;0
+!10 = !{ !"SRV", i32 1, i32 0, i32 5, i32 -1, i32 1 }
+;DESCRIPTORS_VOLATILE
+!11 = !{ !"UAV", i32 5, i32 1, i32 6, i32 5, i32 1 }
+;DATA_VOLATILE
+!12 = !{ !"CBV", i32 5, i32 1, i32 7, i32 5, i32 2 }
+;DATA_STATIC
+!13 = !{ !"SRV", i32 5, i32 1, i32 8, i32 5, i32 8 }
+;DATA_STATIC_WHILE_SET_AT_EXECUTE
+!14 = !{ !"UAV", i32 5, i32 1, i32 9, i32 5, i32 4 }
+;DESCRIPTORS_VOLATILE | DATA_VOLATILE
+!15 = !{ !"CBV", i32 5, i32 1, i32 10, i32 5, i32 3 }
+;DESCRIPTORS_VOLATILE | DATA_STATIC_WHILE_SET_AT_EXECUTE
+!16 = !{ !"SRV", i32 5, i32 1, i32 11, i32 5, i32 5 }
+;DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS
+!17 = !{ !"UAV", i32 5, i32 1, i32 12, i32 5, i32 65536 }
+;DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS | DATA_VOLATILE
+!18 = !{ !"CBV", i32 5, i32 1, i32 13, i32 5, i32 65538 }
+;DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS | DATA_STATIC
+!19 = !{ !"SRV", i32 5, i32 1, i32 14, i32 5, i32 65544 }
+;DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS | DATA_STATIC_WHILE_SET_AT_EXECUTE
+!20 = !{ !"UAV", i32 5, i32 1, i32 15, i32 5, i32 65540 }
+
+;DXC:- Name:            RTS0
+;DXC-NEXT:    Size:            380
+;DXC-NEXT:    RootSignature:
+;DXC-NEXT:      Version:         2
+;DXC-NEXT:      NumRootParameters: 1
+;DXC-NEXT:      RootParametersOffset: 24
+;DXC-NEXT:      NumStaticSamplers: 0
+;DXC-NEXT:      StaticSamplersOffset: 0
+;DXC-NEXT:      Parameters:
+;DXC-NEXT:        - ParameterType:   0
+;DXC-NEXT:          ShaderVisibility: 0
+;DXC-NEXT:          Table:
+;DXC-NEXT:            NumRanges:       14
+;DXC-NEXT:            RangesOffset:    44
+;DXC-NEXT:            Ranges:
+;DXC-NEXT:              - RangeType:       3
+;DXC-NEXT:                NumDescriptors:  1
+;DXC-NEXT:                BaseShaderRegister: 0
+;DXC-NEXT:                RegisterSpace:   1
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 4294967295
+;DXC-NEXT:              - RangeType:       3
+;DXC-NEXT:                NumDescriptors:  1
+;DXC-NEXT:                BaseShaderRegister: 0
+;DXC-NEXT:                RegisterSpace:   3
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 4294967295
+;DXC-NEXT:                DESCRIPTORS_VOLATILE: true
+;DXC-NEXT:              - RangeType:       3
+;DXC-NEXT:                NumDescriptors:  1
+;DXC-NEXT:                BaseShaderRegister: 0
+;DXC-NEXT:                RegisterSpace:   4
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 4294967295
+;DXC-NEXT:                DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS: true
+;DXC-NEXT:              - RangeType:       0
+;DXC-NEXT:                NumDescriptors:  1
+;DXC-NEXT:                BaseShaderRegister: 0
+;DXC-NEXT:                RegisterSpace:   5
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 4294967295
+;DXC-NEXT:                DESCRIPTORS_VOLATILE: true
+;DXC-NEXT:              - RangeType:       1
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   6
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DESCRIPTORS_VOLATILE: true
+;DXC-NEXT:              - RangeType:       2
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   7
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_VOLATILE:   true
+;DXC-NEXT:              - RangeType:       0
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   8
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_STATIC:     true
+;DXC-NEXT:              - RangeType:       1
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   9
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_STATIC_WHILE_SET_AT_EXECUTE: true
+;DXC-NEXT:              - RangeType:       2
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   10
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DESCRIPTORS_VOLATILE: true
+;DXC-NEXT:                DATA_VOLATILE:   true
+;DXC-NEXT:              - RangeType:       0
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   11
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DESCRIPTORS_VOLATILE: true
+;DXC-NEXT:                DATA_STATIC_WHILE_SET_AT_EXECUTE: true
+;DXC-NEXT:              - RangeType:       1
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   12
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS: true
+;DXC-NEXT:              - RangeType:       2
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   13
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_VOLATILE:   true
+;DXC-NEXT:                DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS: true
+;DXC-NEXT:              - RangeType:       0
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   14
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_STATIC:     true
+;DXC-NEXT:                DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS: true
+;DXC-NEXT:              - RangeType:       1
+;DXC-NEXT:                NumDescriptors:  5
+;DXC-NEXT:                BaseShaderRegister: 1
+;DXC-NEXT:                RegisterSpace:   15
+;DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+;DXC-NEXT:                DATA_STATIC_WHILE_SET_AT_EXECUTE: true
+;DXC-NEXT:                DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS: true
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinationsV1.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinationsV1.ll
new file mode 100644
index 000000000000..9d89dbdd9107
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-AllValidFlagCombinationsV1.ll
@@ -0,0 +1,44 @@
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 1 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !7 }
+!6 = !{ !"Sampler", i32 0, i32 1, i32 0, i32 -1, i32 1 }
+!7 = !{ !"UAV", i32 5, i32 1, i32 10, i32 5, i32 3 }
+
+
+; DXC:        - Name:            RTS0
+; DXC-NEXT:     Size:            84
+; DXC-NEXT:     RootSignature:
+; DXC-NEXT:       Version:         1
+; DXC-NEXT:       NumRootParameters: 1
+; DXC-NEXT:       RootParametersOffset: 24
+; DXC-NEXT:       NumStaticSamplers: 0
+; DXC-NEXT:       StaticSamplersOffset: 0
+; DXC-NEXT:       Parameters:
+; DXC-NEXT:         - ParameterType:   0
+; DXC-NEXT:           ShaderVisibility: 0
+; DXC-NEXT:           Table:
+; DXC-NEXT:             NumRanges:       2
+; DXC-NEXT:             RangesOffset:    44
+; DXC-NEXT:             Ranges:
+; DXC-NEXT:               - RangeType:       3
+; DXC-NEXT:                 NumDescriptors:  0
+; DXC-NEXT:                 BaseShaderRegister: 1
+; DXC-NEXT:                 RegisterSpace:   0
+; DXC-NEXT:                 OffsetInDescriptorsFromTableStart: 4294967295
+; DXC-NEXT:               - RangeType:       1
+; DXC-NEXT:                 NumDescriptors:  5
+; DXC-NEXT:                 BaseShaderRegister: 1
+; DXC-NEXT:                 RegisterSpace:   10
+; DXC-NEXT:                 OffsetInDescriptorsFromTableStart: 5
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-Flag.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-Flag.ll
new file mode 100644
index 000000000000..41101c1f1fe8
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-Flag.ll
@@ -0,0 +1,20 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for DescriptorFlag: 22
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !7 }
+!6 = !{ !"SRV", i32 0, i32 1, i32 0, i32 -1, i32 22 }
+!7 = !{ !"UAV", i32 5, i32 1, i32 10, i32 5, i32 2 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RangeType.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RangeType.ll
new file mode 100644
index 000000000000..b7e99ae7cd27
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RangeType.ll
@@ -0,0 +1,20 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid Descriptor Range type: Invalid 
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !7 }
+!6 = !{ !"Invalid", i32 0, i32 0, i32 -1, i32 -1, i32 4 }
+!7 = !{ !"UAV", i32 5, i32 1, i32 10, i32 5, i32 2 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RegisterSpace.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RegisterSpace.ll
new file mode 100644
index 000000000000..4cef5d86a980
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable-Invalid-RegisterSpace.ll
@@ -0,0 +1,20 @@
+; RUN: not opt -passes='print<dxil-root-signature>' %s -S -o - 2>&1 | FileCheck %s
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: error: Invalid value for RegisterSpace: 4294967280
+; CHECK-NOT: Root Signature Definitions
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !7 }
+!6 = !{ !"SRV", i32 0, i32 0, i32 10, i32 -1, i32 4 }
+!7 = !{ !"UAV", i32 5, i32 1, i32 4294967280, i32 5, i32 2 }
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable.ll
new file mode 100644
index 000000000000..b516d6618024
--- /dev/null
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-DescriptorTable.ll
@@ -0,0 +1,48 @@
+; RUN: opt %s -dxil-embed -dxil-globals -S -o - | FileCheck %s
+; RUN: llc %s --filetype=obj -o - | obj2yaml | FileCheck %s --check-prefix=DXC
+
+target triple = "dxil-unknown-shadermodel6.0-compute"
+
+; CHECK: @dx.rts0 = private constant [92 x i8]  c"{{.*}}", section "RTS0", align 4
+
+define void @main() #0 {
+entry:
+  ret void
+}
+attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
+
+
+!dx.rootsignatures = !{!2} ; list of function/root signature pairs
+!2 = !{ ptr @main, !3, i32 2 } ; function, root signature
+!3 = !{ !5 } ; list of root signature elements
+!5 = !{ !"DescriptorTable", i32 0, !6, !7 }
+!6 = !{ !"SRV", i32 0, i32 1, i32 0, i32 -1, i32 4 }
+!7 = !{ !"UAV", i32 5, i32 1, i32 10, i32 5, i32 2 }
+
+; DXC:  - Name:            RTS0
+; DXC-NEXT:    Size:            92
+; DXC-NEXT:    RootSignature:
+; DXC-NEXT:      Version:         2
+; DXC-NEXT:      NumRootParameters: 1 
+; DXC-NEXT:      RootParametersOffset: 24 
+; DXC-NEXT:      NumStaticSamplers: 0
+; DXC-NEXT:      StaticSamplersOffset: 0
+; DXC-NEXT:      Parameters:
+; DXC-NEXT:        - ParameterType:   0
+; DXC-NEXT:          ShaderVisibility: 0
+; DXC-NEXT:          Table:
+; DXC-NEXT:            NumRanges:       2
+; DXC-NEXT:            RangesOffset:    44
+; DXC-NEXT:            Ranges:
+; DXC-NEXT:              - RangeType:       0
+; DXC-NEXT:                NumDescriptors:  0
+; DXC-NEXT:                BaseShaderRegister: 1
+; DXC-NEXT:                RegisterSpace:   0
+; DXC-NEXT:                OffsetInDescriptorsFromTableStart: 4294967295
+; DXC-NEXT:                DATA_STATIC_WHILE_SET_AT_EXECUTE:   true
+; DXC-NEXT:              - RangeType:       1
+; DXC-NEXT:                NumDescriptors:  5
+; DXC-NEXT:                BaseShaderRegister: 1
+; DXC-NEXT:                RegisterSpace:   10
+; DXC-NEXT:                OffsetInDescriptorsFromTableStart: 5
+; DXC-NEXT:                DATA_VOLATILE: true
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
index 80aa757d7e10..d0a58bc34ffa 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Parameters.ll
@@ -12,16 +12,19 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 
 !dx.rootsignatures = !{!2} ; list of function/root signature pairs
 !2 = !{ ptr @main, !3, i32 2 } ; function, root signature
-!3 = !{ !4, !5, !6 } ; list of root signature elements
+!3 = !{ !4, !5, !6, !7 } ; list of root signature elements
 !4 = !{ !"RootFlags", i32 1 } ; 1 = allow_input_assembler_input_layout
 !5 = !{ !"RootConstants", i32 0, i32 1, i32 2, i32 3 }
 !6 = !{ !"RootSRV", i32 1, i32 4, i32 5, i32 6 }
+!7 = !{ !"DescriptorTable", i32 0, !8, !9 }
+!8 = !{ !"SRV", i32 0, i32 1, i32 0, i32 -1, i32 4 }
+!9 = !{ !"UAV", i32 5, i32 1, i32 10, i32 5, i32 2 }
 
 ;CHECK-LABEL: Definition for 'main':
 ;CHECK-NEXT:  Flags: 0x000001
 ;CHECK-NEXT:  Version: 2
 ;CHECK-NEXT:  RootParametersOffset: 24
-;CHECK-NEXT:  NumParameters: 2
+;CHECK-NEXT:  NumParameters: 3
 ;CHECK-NEXT:   - Parameter Type: 1
 ;CHECK-NEXT:     Shader Visibility: 0
 ;CHECK-NEXT:     Register Space: 2
@@ -32,5 +35,20 @@ attributes #0 = { "hlsl.numthreads"="1,1,1" "hlsl.shader"="compute" }
 ;CHECK-NEXT:     Register Space: 5
 ;CHECK-NEXT:     Shader Register: 4
 ;CHECK-NEXT:     Flags: 6
+;CHECK-NEXT:   - Parameter Type: 0
+;CHECK-NEXT:     Shader Visibility: 0
+;CHECK-NEXT:     NumRanges: 2
+;CHECK-NEXT:     - Range Type: 0
+;CHECK-NEXT:       Register Space: 0
+;CHECK-NEXT:       Base Shader Register: 1
+;CHECK-NEXT:       Num Descriptors: 0
+;CHECK-NEXT:       Offset In Descriptors From Table Start: 4294967295
+;CHECK-NEXT:       Flags: 4
+;CHECK-NEXT:     - Range Type: 1
+;CHECK-NEXT:       Register Space: 10
+;CHECK-NEXT:       Base Shader Register: 1
+;CHECK-NEXT:       Num Descriptors: 5
+;CHECK-NEXT:       Offset In Descriptors From Table Start: 5
+;CHECK-NEXT:       Flags: 2
 ;CHECK-NEXT:  NumStaticSamplers: 0
 ;CHECK-NEXT:  StaticSamplersOffset: 0
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
index 6c90bcb09b64..7ee04710be0a 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-RootDescriptor-Invalid-Flags.ll
@@ -3,7 +3,7 @@
 target triple = "dxil-unknown-shadermodel6.0-compute"
 
 
-; CHECK: error: Invalid value for DescriptorFlag: 3
+; CHECK: error: Invalid value for DescriptorRangeFlag: 3
 ; CHECK-NOT: Root Signature Definitions
 define void @main() #0 {
 entry:
-- 
cgit v1.2.3


From 958dc8602651261f8285b59d352a1c4b4da2e90c Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:21:45 -0700
Subject: [AMDGPU] Don't insert wait instructions that are not supported by
 gfx1250 (#145084)

No tests yet, but it will allow further tests not to be
polluted with these waits.
---
 llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
index d2de494a23ef..3212060f303a 100644
--- a/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMemoryLegalizer.cpp
@@ -2272,8 +2272,10 @@ bool SIGfx12CacheControl::insertWaitsBeforeSystemScopeStore(
   const DebugLoc &DL = MI->getDebugLoc();
 
   BuildMI(MBB, MI, DL, TII->get(S_WAIT_LOADCNT_soft)).addImm(0);
-  BuildMI(MBB, MI, DL, TII->get(S_WAIT_SAMPLECNT_soft)).addImm(0);
-  BuildMI(MBB, MI, DL, TII->get(S_WAIT_BVHCNT_soft)).addImm(0);
+  if (ST.hasImageInsts()) {
+    BuildMI(MBB, MI, DL, TII->get(S_WAIT_SAMPLECNT_soft)).addImm(0);
+    BuildMI(MBB, MI, DL, TII->get(S_WAIT_BVHCNT_soft)).addImm(0);
+  }
   BuildMI(MBB, MI, DL, TII->get(S_WAIT_KMCNT_soft)).addImm(0);
   BuildMI(MBB, MI, DL, TII->get(S_WAIT_STORECNT_soft)).addImm(0);
 
-- 
cgit v1.2.3


From 2a4207e7322c90176814b17870051f9692f9994f Mon Sep 17 00:00:00 2001
From: Farzon Lotfi <farzonlotfi@microsoft.com>
Date: Fri, 20 Jun 2025 15:23:20 -0400
Subject: [DirectX] Don't limit visitGetElementPtrInst to global ptrs (#144959)

fixes #144608
- there is a getPointerOperandIndex function so we don't need to iterate
the operands trying to find the pointer. This resulted in a small
cleanup to visitStoreInst and visitLoadInst.

- The meat of this change was in visitGetElementPtrInst to account for
allocas and not bail when we don't find a global.
---
 llvm/lib/Target/DirectX/DXILDataScalarization.cpp | 104 ++++++++++++----------
 llvm/test/CodeGen/DirectX/scalarize-alloca.ll     |  19 +++-
 2 files changed, 72 insertions(+), 51 deletions(-)

diff --git a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
index 286fd0c5bb97..0a9b2bb99f7e 100644
--- a/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
+++ b/llvm/lib/Target/DirectX/DXILDataScalarization.cpp
@@ -14,11 +14,13 @@
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/ReplaceConstant.h"
 #include "llvm/IR/Type.h"
+#include "llvm/Support/Casting.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/Local.h"
 
@@ -137,49 +139,42 @@ bool DataScalarizerVisitor::visitAllocaInst(AllocaInst &AI) {
 }
 
 bool DataScalarizerVisitor::visitLoadInst(LoadInst &LI) {
-  unsigned NumOperands = LI.getNumOperands();
-  for (unsigned I = 0; I < NumOperands; ++I) {
-    Value *CurrOpperand = LI.getOperand(I);
-    ConstantExpr *CE = dyn_cast<ConstantExpr>(CurrOpperand);
-    if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
-      GetElementPtrInst *OldGEP =
-          cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(LI.getIterator());
-      IRBuilder<> Builder(&LI);
-      LoadInst *NewLoad =
-          Builder.CreateLoad(LI.getType(), OldGEP, LI.getName());
-      NewLoad->setAlignment(LI.getAlign());
-      LI.replaceAllUsesWith(NewLoad);
-      LI.eraseFromParent();
-      visitGetElementPtrInst(*OldGEP);
-      return true;
-    }
-    if (GlobalVariable *NewGlobal = lookupReplacementGlobal(CurrOpperand))
-      LI.setOperand(I, NewGlobal);
+  Value *PtrOperand = LI.getPointerOperand();
+  ConstantExpr *CE = dyn_cast<ConstantExpr>(PtrOperand);
+  if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
+    GetElementPtrInst *OldGEP = cast<GetElementPtrInst>(CE->getAsInstruction());
+    OldGEP->insertBefore(LI.getIterator());
+    IRBuilder<> Builder(&LI);
+    LoadInst *NewLoad = Builder.CreateLoad(LI.getType(), OldGEP, LI.getName());
+    NewLoad->setAlignment(LI.getAlign());
+    LI.replaceAllUsesWith(NewLoad);
+    LI.eraseFromParent();
+    visitGetElementPtrInst(*OldGEP);
+    return true;
   }
+  if (GlobalVariable *NewGlobal = lookupReplacementGlobal(PtrOperand))
+    LI.setOperand(LI.getPointerOperandIndex(), NewGlobal);
   return false;
 }
 
 bool DataScalarizerVisitor::visitStoreInst(StoreInst &SI) {
-  unsigned NumOperands = SI.getNumOperands();
-  for (unsigned I = 0; I < NumOperands; ++I) {
-    Value *CurrOpperand = SI.getOperand(I);
-    ConstantExpr *CE = dyn_cast<ConstantExpr>(CurrOpperand);
-    if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
-      GetElementPtrInst *OldGEP =
-          cast<GetElementPtrInst>(CE->getAsInstruction());
-      OldGEP->insertBefore(SI.getIterator());
-      IRBuilder<> Builder(&SI);
-      StoreInst *NewStore = Builder.CreateStore(SI.getValueOperand(), OldGEP);
-      NewStore->setAlignment(SI.getAlign());
-      SI.replaceAllUsesWith(NewStore);
-      SI.eraseFromParent();
-      visitGetElementPtrInst(*OldGEP);
-      return true;
-    }
-    if (GlobalVariable *NewGlobal = lookupReplacementGlobal(CurrOpperand))
-      SI.setOperand(I, NewGlobal);
+
+  Value *PtrOperand = SI.getPointerOperand();
+  ConstantExpr *CE = dyn_cast<ConstantExpr>(PtrOperand);
+  if (CE && CE->getOpcode() == Instruction::GetElementPtr) {
+    GetElementPtrInst *OldGEP = cast<GetElementPtrInst>(CE->getAsInstruction());
+    OldGEP->insertBefore(SI.getIterator());
+    IRBuilder<> Builder(&SI);
+    StoreInst *NewStore = Builder.CreateStore(SI.getValueOperand(), OldGEP);
+    NewStore->setAlignment(SI.getAlign());
+    SI.replaceAllUsesWith(NewStore);
+    SI.eraseFromParent();
+    visitGetElementPtrInst(*OldGEP);
+    return true;
   }
+  if (GlobalVariable *NewGlobal = lookupReplacementGlobal(PtrOperand))
+    SI.setOperand(SI.getPointerOperandIndex(), NewGlobal);
+
   return false;
 }
 
@@ -302,24 +297,35 @@ bool DataScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
 }
 
 bool DataScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
-
-  unsigned NumOperands = GEPI.getNumOperands();
-  GlobalVariable *NewGlobal = nullptr;
-  for (unsigned I = 0; I < NumOperands; ++I) {
-    Value *CurrOpperand = GEPI.getOperand(I);
-    NewGlobal = lookupReplacementGlobal(CurrOpperand);
-    if (NewGlobal)
-      break;
+  Value *PtrOperand = GEPI.getPointerOperand();
+  Type *OrigGEPType = GEPI.getPointerOperandType();
+  Type *NewGEPType = OrigGEPType;
+  bool NeedsTransform = false;
+
+  if (GlobalVariable *NewGlobal = lookupReplacementGlobal(PtrOperand)) {
+    NewGEPType = NewGlobal->getValueType();
+    PtrOperand = NewGlobal;
+    NeedsTransform = true;
+  } else if (AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrOperand)) {
+    Type *AllocatedType = Alloca->getAllocatedType();
+    // OrigGEPType might just be a pointer lets make sure
+    // to add the allocated type so we have a size
+    if (AllocatedType != OrigGEPType) {
+      NewGEPType = AllocatedType;
+      NeedsTransform = true;
+    }
   }
-  if (!NewGlobal)
+
+  // Note: We bail if this isn't a gep touched via alloca or global
+  // transformations
+  if (!NeedsTransform)
     return false;
 
   IRBuilder<> Builder(&GEPI);
   SmallVector<Value *, MaxVecSize> Indices(GEPI.indices());
 
-  Value *NewGEP =
-      Builder.CreateGEP(NewGlobal->getValueType(), NewGlobal, Indices,
-                        GEPI.getName(), GEPI.getNoWrapFlags());
+  Value *NewGEP = Builder.CreateGEP(NewGEPType, PtrOperand, Indices,
+                                    GEPI.getName(), GEPI.getNoWrapFlags());
   GEPI.replaceAllUsesWith(NewGEP);
   GEPI.eraseFromParent();
   return true;
diff --git a/llvm/test/CodeGen/DirectX/scalarize-alloca.ll b/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
index 4829f3a31791..b589136d6965 100644
--- a/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
+++ b/llvm/test/CodeGen/DirectX/scalarize-alloca.ll
@@ -1,10 +1,25 @@
-; RUN: opt -S -passes='dxil-data-scalarization' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=SCHECK
-; RUN: opt -S -passes='dxil-data-scalarization,dxil-flatten-arrays' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefix=FCHECK
+; RUN: opt -S -passes='dxil-data-scalarization' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=SCHECK,CHECK
+; RUN: opt -S -passes='dxil-data-scalarization,dxil-flatten-arrays' -mtriple=dxil-pc-shadermodel6.3-library %s | FileCheck %s --check-prefixes=FCHECK,CHECK
 
 ; CHECK-LABEL: alloca_2d__vec_test
 define void @alloca_2d__vec_test() local_unnamed_addr #2 {
   ; SCHECK:  alloca [2 x [4 x i32]], align 16
   ; FCHECK:  alloca [8 x i32], align 16
+  ; CHECK: ret void
   %1 = alloca [2 x <4 x i32>], align 16
   ret void
 }
+
+; CHECK-LABEL: alloca_2d_gep_test
+define void @alloca_2d_gep_test() {
+  ; SCHECK:  [[alloca_val:%.*]] = alloca [2 x [2 x i32]], align 16
+  ; FCHECK:  [[alloca_val:%.*]] = alloca [4 x i32], align 16
+  ; CHECK: [[tid:%.*]] = tail call i32 @llvm.dx.thread.id(i32 0)
+  ; SCHECK: [[gep:%.*]] = getelementptr inbounds nuw [2 x [2 x i32]], ptr [[alloca_val]], i32 0, i32 [[tid]]
+  ; FCHECK: [[gep:%.*]] = getelementptr inbounds nuw [4 x i32], ptr [[alloca_val]], i32 0, i32 [[tid]]
+  ; CHECK: ret void
+  %1 = alloca [2 x <2 x i32>], align 16
+  %2 = tail call i32 @llvm.dx.thread.id(i32 0)
+  %3 = getelementptr inbounds nuw [2 x <2 x i32>], ptr %1, i32 0, i32 %2
+  ret void
+}
-- 
cgit v1.2.3


From affcc5e728c86260590ae398c136d43ac6cfbfb0 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:28:18 -0700
Subject: [AMDGPU] Add s_wait_xcnt gfx1250 instruction (#145086)

---
 llvm/lib/Target/AMDGPU/AMDGPU.td                       | 10 ++++++++++
 llvm/lib/Target/AMDGPU/GCNSubtarget.h                  |  5 +++++
 llvm/lib/Target/AMDGPU/SOPInstructions.td              | 10 ++++++++++
 llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s                 | 14 ++++++++++++++
 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt | 10 ++++++++++
 5 files changed, 49 insertions(+)
 create mode 100644 llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
 create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.td b/llvm/lib/Target/AMDGPU/AMDGPU.td
index 559328459141..4b17e1c808b5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.td
@@ -1112,6 +1112,12 @@ def FeaturePointSampleAccel : SubtargetFeature<"point-sample-accel",
   "Has point sample acceleration feature"
 >;
 
+def FeatureWaitXcnt : SubtargetFeature<"wait-xcnt",
+  "HasWaitXcnt",
+  "true",
+  "Has s_wait_xcnt instruction"
+>;
+
 //===------------------------------------------------------------===//
 // Subtarget Features (options and debugging)
 //===------------------------------------------------------------===//
@@ -1927,6 +1933,7 @@ def FeatureISAVersion12_50 : FeatureSet<
    FeatureAshrPkInsts,
    FeatureSupportsSRAMECC,
    FeatureMaxHardClauseLength63,
+   FeatureWaitXcnt,
    FeatureAtomicFMinFMaxF64GlobalInsts,
    FeatureAtomicFMinFMaxF64FlatInsts,
    FeatureFlatBufferGlobalAtomicFaddF64Inst,
@@ -2591,6 +2598,9 @@ def HasPrngInst : Predicate<"Subtarget->hasPrngInst()">,
 def HasBVHDualAndBVH8Insts : Predicate<"Subtarget->hasBVHDualAndBVH8Insts()">,
   AssemblerPredicate<(all_of FeatureBVHDualAndBVH8Insts)>;
 
+def HasWaitXcnt : Predicate<"Subtarget->hasWaitXcnt()">,
+  AssemblerPredicate<(all_of FeatureWaitXcnt)>;
+
 def HasFP8ConversionScaleInsts : Predicate<"Subtarget->hasFP8ConversionScaleInsts()">,
   AssemblerPredicate<(all_of FeatureFP8ConversionScaleInsts)>;
 
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index dd57cc96e41c..4ec60dc2752e 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -202,6 +202,7 @@ protected:
 
   bool HasNoSdstCMPX = false;
   bool HasVscnt = false;
+  bool HasWaitXcnt = false;
   bool HasGetWaveIdInst = false;
   bool HasSMemTimeInst = false;
   bool HasShaderCyclesRegister = false;
@@ -1368,6 +1369,10 @@ public:
     return HasMinimum3Maximum3PKF16;
   }
 
+  /// \returns true if the target has s_wait_xcnt insertion. Supported for
+  /// GFX1250.
+  bool hasWaitXCnt() const { return HasWaitXcnt; }
+
   bool hasPointSampleAccel() const { return HasPointSampleAccel; }
 
   /// \returns The maximum number of instructions that can be enclosed in an
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 90e65a6950c0..3f2e764f2926 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -1751,6 +1751,11 @@ let OtherPredicates = [HasExportInsts] in
                 [(int_amdgcn_s_wait_kmcnt timm:$simm16)]>;
 } // End SubtargetPredicate = isGFX12Plus, hasSideEffects = 1
 
+let SubtargetPredicate = HasWaitXcnt, hasSideEffects = 1 in {
+  def S_WAIT_XCNT :
+    SOPP_Pseudo<"s_wait_xcnt", (ins s16imm:$simm16), "$simm16">;
+} // End SubtargetPredicate = hasWaitXcnt, hasSideEffects = 1
+
 //===----------------------------------------------------------------------===//
 // SOP1 Patterns
 //===----------------------------------------------------------------------===//
@@ -2560,6 +2565,11 @@ defm S_WAIT_KMCNT           : SOPP_Real_32_gfx12<0x047>;
 defm S_WAIT_LOADCNT_DSCNT   : SOPP_Real_32_gfx12<0x048>;
 defm S_WAIT_STORECNT_DSCNT  : SOPP_Real_32_gfx12<0x049>;
 
+//===----------------------------------------------------------------------===//
+// SOPP - GFX1250 only.
+//===----------------------------------------------------------------------===//
+defm S_WAIT_XCNT      : SOPP_Real_32_gfx12<0x045>;
+
 //===----------------------------------------------------------------------===//
 // SOPP - GFX11, GFX12.
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
new file mode 100644
index 000000000000..1aca88771c1f
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx1250_asm_sopp.s
@@ -0,0 +1,14 @@
+// RUN: llvm-mc -triple=amdgcn -show-encoding -mcpu=gfx1250 %s | FileCheck --check-prefix=GFX1250 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: -strict-whitespace %s
+
+s_wait_xcnt 0x0
+// GFX1250: [0x00,0x00,0xc5,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_wait_xcnt 0x7
+// GFX1250: [0x07,0x00,0xc5,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+s_wait_xcnt 0xf
+// GFX1250: [0x0f,0x00,0xc5,0xbf]
+// GFX12-ERR: :[[@LINE-2]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
new file mode 100644
index 000000000000..e785fe9cc6d5
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx1250_dasm_sopp.txt
@@ -0,0 +1,10 @@
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX1250 %s
+
+# GFX1250: s_wait_xcnt 0x0 ; encoding: [0x00,0x00,0xc5,0xbf]
+0x00,0x00,0xc5,0xbf
+
+# GFX1250: s_wait_xcnt 0x7 ; encoding: [0x07,0x00,0xc5,0xbf]
+0x07,0x00,0xc5,0xbf
+
+# GFX1250: s_wait_xcnt 0xf ; encoding: [0x0f,0x00,0xc5,0xbf]
+0x0f,0x00,0xc5,0xbf
-- 
cgit v1.2.3


From 227f759644bbc208045178c54633df241f27da7f Mon Sep 17 00:00:00 2001
From: Maksim Levental <maksim.levental@gmail.com>
Date: Fri, 20 Jun 2025 14:34:43 -0500
Subject: [mlir][python] expose operation.block (#145088)

Expose `operation-getBlock()` in python.
---
 mlir/lib/Bindings/Python/IRCore.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/mlir/lib/Bindings/Python/IRCore.cpp b/mlir/lib/Bindings/Python/IRCore.cpp
index b5720b7ad8b2..cbd35f2974ae 100644
--- a/mlir/lib/Bindings/Python/IRCore.cpp
+++ b/mlir/lib/Bindings/Python/IRCore.cpp
@@ -3385,6 +3385,7 @@ void mlir::python::populateIRCore(nb::module_ &m) {
       .def(MLIR_PYTHON_CAPI_FACTORY_ATTR, &PyOperation::createFromCapsule)
       .def_prop_ro("operation", [](nb::object self) { return self; })
       .def_prop_ro("opview", &PyOperation::createOpView)
+      .def_prop_ro("block", &PyOperation::getBlock)
       .def_prop_ro(
           "successors",
           [](PyOperationBase &self) {
-- 
cgit v1.2.3


From 6d8c6ef90c1a4d17d764c4479d5165251bf07c95 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 20 Jun 2025 12:45:21 -0700
Subject: [BOLT][NFC] Simplify doTrace in BAT mode (#143233)

`BoltAddressTranslation::getFallthroughsInTrace` iterates over address
translation map entries and therefore has direct access to both original
and translated offsets. Return the translated offsets in fall-throughs
list to avoid duplicate address translation inside `doTrace`.

Test Plan: NFC
---
 bolt/lib/Profile/BoltAddressTranslation.cpp | 4 ++--
 bolt/lib/Profile/DataAggregator.cpp         | 7 +------
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/bolt/lib/Profile/BoltAddressTranslation.cpp b/bolt/lib/Profile/BoltAddressTranslation.cpp
index a253522e4fb1..7ad4e6a2e141 100644
--- a/bolt/lib/Profile/BoltAddressTranslation.cpp
+++ b/bolt/lib/Profile/BoltAddressTranslation.cpp
@@ -546,7 +546,7 @@ BoltAddressTranslation::getFallthroughsInTrace(uint64_t FuncAddress,
     return Res;
 
   for (auto Iter = FromIter; Iter != ToIter;) {
-    const uint32_t Src = Iter->first;
+    const uint32_t Src = Iter->second >> 1;
     if (Iter->second & BRANCHENTRY) {
       ++Iter;
       continue;
@@ -557,7 +557,7 @@ BoltAddressTranslation::getFallthroughsInTrace(uint64_t FuncAddress,
       ++Iter;
     if (Iter->second & BRANCHENTRY)
       break;
-    Res.emplace_back(Src, Iter->first);
+    Res.emplace_back(Src, Iter->second >> 1);
   }
 
   return Res;
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index c067b2f5b73b..5c8af3710720 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -835,13 +835,8 @@ bool DataAggregator::doTrace(const Trace &Trace, uint64_t Count,
 
   LLVM_DEBUG(dbgs() << "Processing " << FTs->size() << " fallthroughs for "
                     << FromFunc->getPrintName() << ":" << Trace << '\n');
-  for (auto [From, To] : *FTs) {
-    if (BAT) {
-      From = BAT->translate(FromFunc->getAddress(), From, /*IsBranchSrc=*/true);
-      To = BAT->translate(FromFunc->getAddress(), To, /*IsBranchSrc=*/false);
-    }
+  for (const auto &[From, To] : *FTs)
     doIntraBranch(*ParentFunc, From, To, Count, false);
-  }
 
   return true;
 }
-- 
cgit v1.2.3


From 4959e8a1dadd96499d701bcf02cd9b25dba98c98 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 20 Jun 2025 12:46:56 -0700
Subject: [BOLT][NFCI] Use heuristic for matching split global functions
 (#90429)

This change speeds up fragment matching for large BOLTed binaries where
all fragments of global parent functions are put under `bolt-pseudo.o`
file symbol:
- before: iterating over symbols under `bolt-pseudo.o` only to fail
  to find a parent,
- after: bail out immediately and use a global parent by name.

Test Plan: NFC, updated register-fragments-bolt-symbols.s
---
 bolt/lib/Rewrite/RewriteInstance.cpp            | 5 +++++
 bolt/test/X86/register-fragments-bolt-symbols.s | 7 +++++++
 2 files changed, 12 insertions(+)

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index e1aa00a3d749..d650e5db54bf 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -1567,6 +1567,11 @@ void RewriteInstance::registerFragments() {
 
     uint64_t ParentAddress{0};
 
+    // Check if containing FILE symbol is BOLT emitted synthetic symbol marking
+    // local fragments of global parents.
+    if (cantFail(FSI[-1].getName()) == getBOLTFileSymbolName())
+      goto registerParent;
+
     // BOLT split fragment symbols are emitted just before the main function
     // symbol.
     for (ELFSymbolRef NextSymbol = Symbol; NextSymbol < StopSymbol;
diff --git a/bolt/test/X86/register-fragments-bolt-symbols.s b/bolt/test/X86/register-fragments-bolt-symbols.s
index c9f1859c4e8a..20e7345541d9 100644
--- a/bolt/test/X86/register-fragments-bolt-symbols.s
+++ b/bolt/test/X86/register-fragments-bolt-symbols.s
@@ -29,6 +29,7 @@
 
 # RUN: link_fdata %s %t.bolt %t.preagg PREAGG
 # PREAGG: B X:0 #chain.cold.0# 1 0
+# PREAGG: B X:0 #dummy# 1 0
 # RUN: perf2bolt %t.bolt -p %t.preagg --pa -o %t.bat.fdata -w %t.bat.yaml -v=1 \
 # RUN:   | FileCheck %s --check-prefix=CHECK-REGISTER
 # RUN: FileCheck --input-file %t.bat.fdata --check-prefix=CHECK-FDATA %s
@@ -44,7 +45,13 @@
 # CHECK-SYMS: l  F .text.cold     [[#]] chain.cold.0
 # CHECK-SYMS: l  F .text          [[#]] chain
 # CHECK-SYMS: l df *ABS*          [[#]] bolt-pseudo.o
+# CHECK-SYMS: l  F .text.cold     [[#]] dummy.cold.0
+# CHECK-SYMS: l  F .text.cold.1   [[#]] dummy.cold.1
+# CHECK-SYMS: l  F .text.cold.2   [[#]] dummy.cold.2
 
+# CHECK-REGISTER: BOLT-INFO: marking dummy.cold.0/1(*2) as a fragment of dummy
+# CHECK-REGISTER: BOLT-INFO: marking dummy.cold.1/1(*2) as a fragment of dummy
+# CHECK-REGISTER: BOLT-INFO: marking dummy.cold.2/1(*2) as a fragment of dummy
 # CHECK-REGISTER: BOLT-INFO: marking chain.cold.0/1(*2) as a fragment of chain/2(*2)
 
 # CHECK-FDATA: 0 [unknown] 0 1 chain/chain.s/2 10 0 1
-- 
cgit v1.2.3


From d8924d4da78fc980b720b328897b1bd5efba348a Mon Sep 17 00:00:00 2001
From: nerix <nerixdev@outlook.de>
Date: Fri, 20 Jun 2025 21:54:06 +0200
Subject: [LLDB] Explicitly use python for version fixup (#144217)

On Windows, the post build command would open the script in the default
editor, since it doesn't know about shebangs. This effectively adds
`python3` in front of the command.

Amends https://github.com/llvm/llvm-project/pull/142871 /
https://github.com/llvm/llvm-project/pull/141116
---
 lldb/source/API/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/API/CMakeLists.txt b/lldb/source/API/CMakeLists.txt
index 2772aa920411..f4a323aaf267 100644
--- a/lldb/source/API/CMakeLists.txt
+++ b/lldb/source/API/CMakeLists.txt
@@ -326,7 +326,7 @@ foreach(header
 endforeach()
 
 add_custom_command(TARGET liblldb POST_BUILD
-  COMMAND ${LLDB_SOURCE_DIR}/scripts/version-header-fix.py ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h ${lldb_header_staging_dir}/lldb-defines.h ${LLDB_VERSION_MAJOR} ${LLDB_VERSION_MINOR} ${LLDB_VERSION_PATCH}
+  COMMAND "${Python3_EXECUTABLE}" ${LLDB_SOURCE_DIR}/scripts/version-header-fix.py ${LLDB_SOURCE_DIR}/include/lldb/lldb-defines.h ${lldb_header_staging_dir}/lldb-defines.h ${LLDB_VERSION_MAJOR} ${LLDB_VERSION_MINOR} ${LLDB_VERSION_PATCH}
 )
 add_custom_target(liblldb-header-staging DEPENDS ${lldb_staged_headers})
 add_dependencies(liblldb liblldb-header-staging)
-- 
cgit v1.2.3


From 3a66e2065296b3e0b27f0a14431eba1d74e7f8c4 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 12:57:36 -0700
Subject: [AMDGPU] Add gfx1250 runlines to vop3 dpp tests. NFC. (#145089)

dpp8 disasm test does not work yet.
---
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s         | 104 ++++++++++++-------
 llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s          |  86 +++++++++++-----
 .../Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt  | 113 ++++++++++++---------
 3 files changed, 195 insertions(+), 108 deletions(-)

diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index cf497c058b7e..90a449173320 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -1,8 +1,10 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1200,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1200,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0]
 // GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -1237,37 +1239,37 @@ v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_m
 // GFX12: v_cvt_pk_bf8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
 v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
 v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| quad_perm:[0,1,2,3]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v255.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xff,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xe4,0x00,0xff]
 
 v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd
 // GFX12: v_cvt_sr_bf8_f32_e64_dpp v1, -v2, v3 quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x00,0x6c,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
@@ -5592,34 +5594,67 @@ v_pack_b32_f16_e64_dpp v255, -|v255.l|, -|v255.h| op_sel:[0,1,0] row_xmask:15 ro
 // GFX12: v_pack_b32_f16_e64_dpp v255, -|v255.l|, -|v255.h| op_sel:[0,1,0] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
 
 v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+
+v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+
+v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX1200: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
 
 v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
 // GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
 
-v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x00,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX12-ERR: :[[@LINE-1]]:45: error: op_sel operand conflicts with 16-bit operand suffix
 
-v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x04,0x00]
+v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
 v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+
+v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
 
 v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
 v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
+// GFX1200: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd6,0xfa,0x04,0xc2,0x03,0x01,0x1b,0x00,0xff]
 
 v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h quad_perm:[0,1,2,3]
-// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+// GFX1200: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+// GFX1200: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
 // GFX12-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
@@ -5627,35 +5662,32 @@ v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 ban
 v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
 // GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
 
-v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0c,0x04,0x01,0xe4,0x00,0x00]
-
 v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
 v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
 
 v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h row_mirror
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
-// GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
 
 v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
-// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[2,2,3,1] row_mask:0xf bank_mask:0xf fi:1 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x7a,0x04,0xff]
 
 v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
-// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[1,1,0] neg_hi:[1,0,1] quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xe ; encoding: [0x00,0x45,0x1a,0xcc,0xfa,0x04,0x0e,0x7c,0x01,0x1b,0x00,0xfe]
 
 v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
-// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0xf bank_mask:0xf ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0xff]
 
 v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
-// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[1,2,3,0] row_mask:0x0 bank_mask:0x0 ; encoding: [0x00,0x40,0x1a,0xcc,0xfa,0x04,0x0e,0x1c,0x01,0x39,0x00,0x00]
 
 v_minimum_f32 v5, v1, v2 quad_perm:[3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index f43a1128f521..a73aa40a2751 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -1,8 +1,10 @@
 // NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --unique --version 5
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
-// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1200,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,GFX1200,W64 %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize32,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W32-ERR --implicit-check-not=error: %s
 // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -filetype=null %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR,W64-ERR --implicit-check-not=error: %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize32,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W32 %s
+// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+wavefrontsize64,+real-true16 -show-encoding %s | FileCheck --check-prefixes=GFX12,W64 %s
 
 v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x55,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
@@ -761,19 +763,19 @@ v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,
 // GFX12: v_cubetc_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x0e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
 
 v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,2,3,0,1] ; encoding: [0x05,0x00,0x69,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0xa9,0x21]
 
 v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v5.l, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x69,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
 
 v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v5.l, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x69,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
 
 v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v255.l, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0]
-// GFX12: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+// GFX1200: v_cvt_pk_fp8_f32_e64_dpp v255.h, -|v255|, -|v255| op_sel:[0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x43,0x69,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_cvt_pk_bf8_f32_e64_dpp v5.l, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6a,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
@@ -3849,34 +3851,70 @@ v_pack_b32_f16_e64_dpp v255, -|v255.l|, -|v255.h| op_sel:[0,1,0] dpp8:[0,0,0,0,0
 // GFX12: v_pack_b32_f16_e64_dpp v255, -|v255.l|, -|v255.h| op_sel:[0,1,0] dpp8:[0,0,0,0,0,0,0,0] fi:1 ; encoding: [0xff,0x13,0x11,0xd7,0xea,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
 
 v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
-v_dot2_f16_f16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:39: error: invalid op_sel operand
+
+v_dot2_f16_f16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:28: error: invalid operand for instruction
+
+v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+
+v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1200: v_dot2_f16_f16_e64_dpp v5, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
+// GFX12-ERR: :[[@LINE-1]]:41: error: invalid op_sel operand
+
+v_dot2_bf16_bf16_e64_dpp v0, s1, v2, v3 dpp8:[0,1,2,3,4,4,4,4]
 // GFX12-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction
 
+v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, v1, s2, v3 dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+
 v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4,4]
 // GFX12-ERR: :[[@LINE-1]]:43: error: invalid op_sel operand
 
+v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
 v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x66,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
 v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x66,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+
+v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
 
 v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x66,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
 v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, 0.5 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd6,0xe9,0x04,0xc2,0x03,0x01,0x77,0x39,0x05]
 
 v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x66,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x66,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+// GFX1200: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x66,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 v_dot2_bf16_bf16_e64_dpp v0.l, s1, v2, v3.l dpp8:[0,1,2,3,4,4,4,4]
 // GFX12-ERR: :[[@LINE-1]]:32: error: invalid operand for instruction
@@ -3885,28 +3923,28 @@ v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l op_sel:[1,1,0,0] dpp8:[0,1,2,3,4,4,4
 // GFX12-ERR: :[[@LINE-1]]:45: error: invalid op_sel operand
 
 v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.l, v1, s2, v3.l dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x00,0x67,0xd6,0xe9,0x04,0x0c,0x04,0x01,0x88,0x46,0x92]
 
 v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x60,0x67,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x88,0x46,0x92]
 
 v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 
 v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
 
 v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h dpp8:[7,6,5,4,3,2,1,0]
-// GFX12: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x20,0x67,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
 
 v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| dpp8:[0,0,0,0,0,0,0,0] fi:0
-// GFX12: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+// GFX1200: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x47,0x67,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
 
 v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
-// GFX12: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v5, v1, v2, s3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x40,0x1a,0xcc,0xe9,0x04,0x0e,0x18,0x01,0x77,0x39,0x05]
 
 v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
-// GFX12: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
+// GFX1200: v_dot2_f32_bf16_e64_dpp v0, v1, v2, v3 neg_lo:[0,1,1] neg_hi:[1,0,1] dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x00,0x45,0x1a,0xcc,0xe9,0x04,0x0e,0xdc,0x01,0x77,0x39,0x05]
 
 v_minimum_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index 397852da6ef3..19915341a4be 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -1,8 +1,10 @@
 # NOTE: Assertions have been autogenerated by utils/update_mc_test_checks.py UTC_ARGS: --version 5
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16 %s
-# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-REAL16,GFX1200,GFX1200-W32,GFX1200-W32-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W32,W32-FAKE16,GFX1200,GFX1200-W32,GFX1200-W32-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-REAL16,GFX1200,GFX1200-W64,GFX1200-W64-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -mattr=+wavefrontsize64,-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,W64,W64-FAKE16,GFX1200,GFX1200-W64,GFX1200-W64-FAKE16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=+real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX1250,W32,GFX1250-W32,W32-REAL16,GFX1250-W32-REAL16 %s
+# RUN: llvm-mc -triple=amdgcn -mcpu=gfx1250 -mattr=-real-true16 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12,GFX1250,W32,GFX1250-W32,W32-FAKE16,GFX1250-W32-FAKE16 %s
 
 0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # GFX12: v_add3_u32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x55,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -1179,62 +1181,62 @@
 # W64-FAKE16: v_cvt_pk_bf8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x6a,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20
-# W32-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 # W32-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 # W64-REAL16: v_cvt_pk_fp8_f32 v1.l, -v2, |v3|        ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 # W64-FAKE16: v_cvt_pk_fp8_f32 v1, -v2, |v3|          ; encoding: [0x01,0x02,0x69,0xd7,0x02,0x07,0x02,0x20]
 
 0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v6.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v6, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x06,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v6, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x06,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v255| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0xfe,0x03,0x20,0x02,0x1b,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[0,2,1,3] row_mask:0xe bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0xd8,0x00,0xed]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0x2 bank_mask:0xd ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0x2d]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0x5 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x00,0xe5]
 
 0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.l, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x02,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
 0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed
-# W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W32-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
+# GFX1200-W32-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W64-REAL16: v_cvt_pk_fp8_f32_e64_dpp v1.h, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 # W64-FAKE16: v_cvt_pk_fp8_f32_e64_dpp v1, -v2, |v3| op_sel:[0,0,1] quad_perm:[3,2,1,0] row_mask:0xe bank_mask:0xd fi:1 ; encoding: [0x01,0x42,0x69,0xd7,0xfa,0x06,0x02,0x20,0x02,0x1b,0x04,0xed]
 
@@ -5793,7 +5795,6 @@
 # W64-REAL16: v_max3_num_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x13,0x2c,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_max3_num_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x13,0x2c,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
 
-
 0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_max3_i16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_max3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -6628,7 +6629,6 @@
 # W64-REAL16: v_min3_num_f16_e64_dpp v5.l, -|v1.l|, -|v2.h|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x13,0x2b,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
 # W64-FAKE16: v_min3_num_f16_e64_dpp v5, -|v1|, -|v2|, 0.5 op_sel:[0,1,0,0] row_xmask:0 row_mask:0x1 bank_mask:0x3 ; encoding: [0x05,0x13,0x2b,0xd6,0xfa,0x04,0xc2,0x63,0x01,0x60,0x01,0x13]
 
-
 0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
 # W32-REAL16: v_min3_i16_e64_dpp v5.l, v1.l, v2.l, v3.l quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
 # W32-FAKE16: v_min3_i16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x4a,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
@@ -7290,100 +7290,108 @@
 # W64-FAKE16: v_sub_nc_u16_e64_dpp v255, v255, v255 op_sel:[0,0,1] clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0xc0,0x04,0xd7,0xfa,0xfe,0x03,0x00,0xff,0x6f,0x0d,0x30]
 
 0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 # op_sel[1:0] are ignored
 0x00,0x78,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x66,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x66,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
 0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1250: v_add_f32_dpp v255, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1250: v_lshlrev_b32_e32 v6, v255, v183        ; encoding: [0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1250: v_add_f32_dpp v255, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x66,0xd6,0xfa,0x04,0xfe,0x07,0x01,0xe4,0x00,0xff]
 
 0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
-# W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1250: v_lshlrev_b32_e32 v6, v255, v183        ; encoding: [0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_dot2_f16_f16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_dot2_f16_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x66,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.l, v1, v2, v3.l quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x00,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 # op_sel[1:0] are ignored
 0x00,0x78,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, v1, v2, v3.h op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, v1, v2, v3 op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x60,0x67,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x04,0x00]
 
 0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v0.h, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 
 0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1250: v_add_f32_dpp v255, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
 0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1250: v_lshlrev_b32_e32 v6, v255, v183        ; encoding: [0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
 0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1250: v_add_f32_dpp v255, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v5.l, v1, v2, v255.h op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v5, v1, v2, v255 op_sel:[0,0,1,0] row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x20,0x67,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x40,0x01,0xff]
 
 0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
-# W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
-# W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1250: v_lshlrev_b32_e32 v6, v255, v183        ; encoding: [0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+# GFX1200-W32-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-REAL16: v_dot2_bf16_bf16_e64_dpp v255.h, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 # W64-FAKE16: v_dot2_bf16_bf16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x47,0x67,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
 
@@ -8287,3 +8295,12 @@
 # W64-REAL16: v_minimummaximum_f16_e64_dpp v255.h, -|v255.l|, -|v255.l|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 # W64-FAKE16: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| op_sel:[0,0,0,1] clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 fi:1 ; encoding: [0xff,0xc7,0x6e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
 
+## NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+# GFX1200: {{.*}}
+# GFX1200-W32: {{.*}}
+# GFX1200-W64: {{.*}}
+# GFX1200-W64-FAKE16: {{.*}}
+# GFX1200-W64-REAL16: {{.*}}
+# GFX1250-W32: {{.*}}
+# GFX1250-W32-FAKE16: {{.*}}
+# GFX1250-W32-REAL16: {{.*}}
-- 
cgit v1.2.3


From d078ce7c98a3f9371d01d526e20f671ca2231667 Mon Sep 17 00:00:00 2001
From: sribee8 <sriya.pratipati@gmail.com>
Date: Fri, 20 Jun 2025 13:00:59 -0700
Subject: [libc] mbrtowc implementation (#144760)

implemented the internal and public mbrtowc as well as tests for the
public function.

---------

Co-authored-by: Sriya Pratipati <sriyap@google.com>
---
 libc/config/linux/x86_64/entrypoints.txt |   3 +
 libc/hdr/types/CMakeLists.txt            |   8 ++
 libc/hdr/types/mbstate_t.h               |  22 ++++
 libc/include/llvm-libc-types/mbstate_t.h |   6 +-
 libc/include/wchar.yaml                  |   9 ++
 libc/src/__support/wchar/CMakeLists.txt  |  16 +++
 libc/src/__support/wchar/mbrtowc.cpp     |  49 +++++++++
 libc/src/__support/wchar/mbrtowc.h       |  29 ++++++
 libc/src/__support/wchar/mbstate.h       |   6 +-
 libc/src/wchar/CMakeLists.txt            |  17 +++
 libc/src/wchar/mbrtowc.cpp               |  38 +++++++
 libc/src/wchar/mbrtowc.h                 |  24 +++++
 libc/test/src/wchar/CMakeLists.txt       |  14 +++
 libc/test/src/wchar/mbrtowc_test.cpp     | 172 +++++++++++++++++++++++++++++++
 14 files changed, 409 insertions(+), 4 deletions(-)
 create mode 100644 libc/hdr/types/mbstate_t.h
 create mode 100644 libc/src/__support/wchar/mbrtowc.cpp
 create mode 100644 libc/src/__support/wchar/mbrtowc.h
 create mode 100644 libc/src/wchar/mbrtowc.cpp
 create mode 100644 libc/src/wchar/mbrtowc.h
 create mode 100644 libc/test/src/wchar/mbrtowc_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index aa2079faed40..8bf6c402b039 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -1244,6 +1244,9 @@ if(LLVM_LIBC_FULL_BUILD)
     libc.src.sys.socket.recv
     libc.src.sys.socket.recvfrom
     libc.src.sys.socket.recvmsg
+
+    # wchar.h entrypoints
+    libc.src.wchar.mbrtowc
   )
 endif()
 
diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt
index c88c35700907..e4b3cb0faa82 100644
--- a/libc/hdr/types/CMakeLists.txt
+++ b/libc/hdr/types/CMakeLists.txt
@@ -20,6 +20,14 @@ add_proxy_header_library(
     libc.include.uchar
 )
 
+add_proxy_header_library(
+  mbstate_t
+  HDRS
+    mbstate_t.h
+  DEPENDS
+    libc.include.llvm-libc-types.mbstate_t
+)
+
 add_proxy_header_library(
   div_t
   HDRS
diff --git a/libc/hdr/types/mbstate_t.h b/libc/hdr/types/mbstate_t.h
new file mode 100644
index 000000000000..367c6af7a3ff
--- /dev/null
+++ b/libc/hdr/types/mbstate_t.h
@@ -0,0 +1,22 @@
+//===-- Definition of macros from mbstate_t.h -----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_HDR_TYPES_MBSTATE_T_H
+#define LLVM_LIBC_HDR_TYPES_MBSTATE_T_H
+
+#ifdef LIBC_FULL_BUILD
+
+#include "include/llvm-libc-types/mbstate_t.h"
+
+#else // Overlay mode
+
+#error "Cannot overlay mbstate_t
+
+#endif // LLVM_LIBC_FULL_BUILD
+
+#endif // LLVM_LIBC_HDR_TYPES_MBSTATE_T_H
diff --git a/libc/include/llvm-libc-types/mbstate_t.h b/libc/include/llvm-libc-types/mbstate_t.h
index 540d50975a26..009fe57da50e 100644
--- a/libc/include/llvm-libc-types/mbstate_t.h
+++ b/libc/include/llvm-libc-types/mbstate_t.h
@@ -9,8 +9,12 @@
 #ifndef LLVM_LIBC_TYPES_MBSTATE_T_H
 #define LLVM_LIBC_TYPES_MBSTATE_T_H
 
-// TODO: Complete this once we implement functions that operate on this type.
+#include "../llvm-libc-macros/stdint-macros.h"
+
 typedef struct {
+  uint32_t __field1;
+  uint8_t __field2;
+  uint8_t __field3;
 } mbstate_t;
 
 #endif // LLVM_LIBC_TYPES_MBSTATE_T_H
diff --git a/libc/include/wchar.yaml b/libc/include/wchar.yaml
index 84db73d8f01e..c036636e12c3 100644
--- a/libc/include/wchar.yaml
+++ b/libc/include/wchar.yaml
@@ -29,6 +29,15 @@ functions:
     return_type: wint_t
     arguments:
       - type: int
+  - name: mbrtowc
+    standards:
+      - stdc
+    return_type: size_t
+    arguments:
+      - type: wchar_t *__restrict
+      - type: const char *__restrict
+      - type: size_t
+      - type: mbstate_t *__restrict
   - name: wmemset
     standards:
       - stdc
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index 6715e354e23e..479c1dff2c6e 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -19,3 +19,19 @@ add_object_library(
     libc.src.__support.math_extras
     .mbstate
 )
+
+add_object_library(
+  mbrtowc
+  HDRS
+    mbrtowc.h
+  SRCS
+    mbrtowc.cpp
+  DEPENDS
+  libc.hdr.types.wchar_t
+  libc.hdr.types.size_t
+  libc.src.__support.common
+  libc.src.__support.error_or
+  libc.src.__support.macros.config
+  .character_converter
+  .mbstate
+)
diff --git a/libc/src/__support/wchar/mbrtowc.cpp b/libc/src/__support/wchar/mbrtowc.cpp
new file mode 100644
index 000000000000..954c7458f4df
--- /dev/null
+++ b/libc/src/__support/wchar/mbrtowc.cpp
@@ -0,0 +1,49 @@
+//===-- Implementation for mbrtowc function ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/wchar/mbrtowc.h"
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict s,
+                        size_t n, mbstate *__restrict ps) {
+  CharacterConverter char_conv(ps);
+  if (s == nullptr)
+    return 0;
+  size_t i = 0;
+  // Reading in bytes until we have a complete wc or error
+  for (; i < n && !char_conv.isFull(); ++i) {
+    int err = char_conv.push(static_cast<char8_t>(s[i]));
+    // Encoding error
+    if (err == -1)
+      return Error(-1);
+  }
+  auto wc = char_conv.pop_utf32();
+  if (wc.has_value()) {
+    *pwc = wc.value();
+    // null terminator -> return 0
+    if (wc.value() == L'\0')
+      return 0;
+    return i;
+  }
+  // Incomplete but potentially valid
+  return -2;
+}
+
+} // namespace internal
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/mbrtowc.h b/libc/src/__support/wchar/mbrtowc.h
new file mode 100644
index 000000000000..37329ee61bea
--- /dev/null
+++ b/libc/src/__support/wchar/mbrtowc.h
@@ -0,0 +1,29 @@
+//===-- Implementation header for mbrtowc function --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC___SUPPORT_WCHAR_MBRTOWC
+#define LLVM_LIBC_SRC___SUPPORT_WCHAR_MBRTOWC
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+ErrorOr<size_t> mbrtowc(wchar_t *__restrict pwc, const char *__restrict s,
+                        size_t n, mbstate *__restrict ps);
+
+} // namespace internal
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC___SUPPORT_WCHAR_MBRTOWC
diff --git a/libc/src/__support/wchar/mbstate.h b/libc/src/__support/wchar/mbstate.h
index fea693f73c3b..32304a521524 100644
--- a/libc/src/__support/wchar/mbstate.h
+++ b/libc/src/__support/wchar/mbstate.h
@@ -18,17 +18,17 @@ namespace internal {
 
 struct mbstate {
   // store a partial codepoint (in UTF-32)
-  char32_t partial;
+  char32_t partial = 0;
 
   /*
   Progress towards a conversion
     Increases with each push(...) until it reaches total_bytes
     Decreases with each pop(...) until it reaches 0
   */
-  uint8_t bytes_stored;
+  uint8_t bytes_stored = 0;
 
   // Total number of bytes that will be needed to represent this character
-  uint8_t total_bytes;
+  uint8_t total_bytes = 0;
 };
 
 } // namespace internal
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index 491dd5b34340..163c29847e6a 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -34,6 +34,23 @@ add_entrypoint_object(
     libc.src.__support.wctype_utils
 )
 
+add_entrypoint_object(
+  mbrtowc
+  SRCS
+    mbrtowc.cpp
+  HDRS
+    mbrtowc.h
+  DEPENDS
+    libc.hdr.types.size_t
+    libc.hdr.types.mbstate_t
+    libc.hdr.types.wchar_t
+    libc.src.__support.common
+    libc.src.__support.macros.config
+    libc.src.__support.wchar.mbrtowc
+    libc.src.__support.libc_errno
+    libc.src.__support.wchar.mbstate
+)
+
 add_entrypoint_object(
   wmemset
   SRCS
diff --git a/libc/src/wchar/mbrtowc.cpp b/libc/src/wchar/mbrtowc.cpp
new file mode 100644
index 000000000000..cd429ab8d30e
--- /dev/null
+++ b/libc/src/wchar/mbrtowc.cpp
@@ -0,0 +1,38 @@
+//===-- Implementation of mbrtowc -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/mbrtowc.h"
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/mbrtowc.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(size_t, mbrtowc,
+                   (wchar_t *__restrict pwc, const char *__restrict s, size_t n,
+                    mbstate_t *__restrict ps)) {
+  static internal::mbstate internal_mbstate;
+  auto ret = internal::mbrtowc(pwc, s, n,
+                               ps == nullptr
+                                   ? &internal_mbstate
+                                   : reinterpret_cast<internal::mbstate *>(ps));
+  if (!ret.has_value()) {
+    // Encoding failure
+    libc_errno = EILSEQ;
+    return -1;
+  }
+  return ret.value();
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/mbrtowc.h b/libc/src/wchar/mbrtowc.h
new file mode 100644
index 000000000000..e2e3d3ebd285
--- /dev/null
+++ b/libc/src/wchar/mbrtowc.h
@@ -0,0 +1,24 @@
+//===-- Implementation header for mbrtowc ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_MBRTOWC_H
+#define LLVM_LIBC_SRC_WCHAR_MBRTOWC_H
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t mbrtowc(wchar_t *__restrict pwc, const char *__restrict s, size_t n,
+               mbstate_t *__restrict ps);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_MBRTOWC_H
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index 4990b6953348..d4cae1f6228b 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -23,6 +23,20 @@ add_libc_test(
     libc.src.wchar.btowc
 )
 
+add_libc_test(
+  mbrtowc_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    mbrtowc_test.cpp
+  DEPENDS
+    libc.src.__support.libc_errno
+    libc.src.string.memset
+    libc.src.wchar.mbrtowc
+    libc.hdr.types.mbstate_t
+    libc.hdr.types.wchar_t
+)
+
 add_libc_test(
   wctob_test
   SUITE
diff --git a/libc/test/src/wchar/mbrtowc_test.cpp b/libc/test/src/wchar/mbrtowc_test.cpp
new file mode 100644
index 000000000000..69dcf00fde20
--- /dev/null
+++ b/libc/test/src/wchar/mbrtowc_test.cpp
@@ -0,0 +1,172 @@
+//===-- Unittests for mbrtowc ---------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/libc_errno.h"
+#include "src/string/memset.h"
+#include "src/wchar/mbrtowc.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcMBRToWC, OneByte) {
+  const char *ch = "A";
+  wchar_t dest[2];
+  // Testing if it works with nullptr mbstate_t
+  mbstate_t *mb = nullptr;
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<char>(*dest), 'A');
+  ASSERT_EQ(static_cast<int>(n), 1);
+
+  // Should fail since we have not read enough
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 0, mb);
+  ASSERT_EQ(static_cast<int>(n), -2);
+}
+
+TEST(LlvmLibcMBRToWC, TwoByte) {
+  const char ch[2] = {static_cast<char>(0xC2),
+                      static_cast<char>(0x8E)}; // Ž car symbol
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  ASSERT_EQ(static_cast<int>(*dest), 142);
+  ASSERT_EQ(static_cast<int>(n), 2);
+
+  // Should fail since we have not read enough
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), -2);
+  // Should pass after reading one more byte
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), 1);
+  ASSERT_EQ(static_cast<int>(*dest), 142);
+}
+
+TEST(LlvmLibcMBRToWC, ThreeByte) {
+  const char ch[3] = {static_cast<char>(0xE2), static_cast<char>(0x88),
+                      static_cast<char>(0x91)}; // ∑ sigma symbol
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
+  ASSERT_EQ(static_cast<int>(*dest), 8721);
+  ASSERT_EQ(static_cast<int>(n), 3);
+
+  // Should fail since we have not read enough
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), -2);
+  // Should pass after reading two more bytes
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_EQ(static_cast<int>(*dest), 8721);
+}
+
+TEST(LlvmLibcMBRToWC, FourByte) {
+  const char ch[4] = {static_cast<char>(0xF0), static_cast<char>(0x9F),
+                      static_cast<char>(0xA4),
+                      static_cast<char>(0xA1)}; // 🤡 clown emoji
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  ASSERT_EQ(static_cast<int>(*dest), 129313);
+  ASSERT_EQ(static_cast<int>(n), 4);
+
+  // Should fail since we have not read enough
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), -2);
+  // Should pass after reading two more bytes
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 2, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_EQ(static_cast<int>(*dest), 129313);
+}
+
+TEST(LlvmLibcMBRToWC, InvalidByte) {
+  const char ch[1] = {static_cast<char>(0x80)};
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), -1);
+  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+}
+
+TEST(LlvmLibcMBRToWC, InvalidMultiByte) {
+  const char ch[4] = {static_cast<char>(0x80), static_cast<char>(0x00),
+                      static_cast<char>(0x80),
+                      static_cast<char>(0x00)}; // invalid sequence of bytes
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  // Trying to push all 4 should error
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  ASSERT_EQ(static_cast<int>(n), -1);
+  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  // Trying to push just the first one should error
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), -1);
+  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+  // Trying to push the second and third should correspond to null wc
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch + 1, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 0);
+  ASSERT_TRUE(*dest == L'\0');
+}
+
+TEST(LlvmLibcMBRToWC, InvalidLastByte) {
+  // Last byte is invalid since it does not have correct starting sequence.
+  // 0xC0 --> 11000000 starting sequence should be 10xxxxxx
+  const char ch[4] = {static_cast<char>(0xF1), static_cast<char>(0x80),
+                      static_cast<char>(0x80), static_cast<char>(0xC0)};
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  // Trying to push all 4 should error
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 4, mb);
+  ASSERT_EQ(static_cast<int>(n), -1);
+  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+}
+
+TEST(LlvmLibcMBRToWC, ValidTwoByteWithExtraRead) {
+  const char ch[3] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+                      static_cast<char>(0x80)};
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  // Trying to push all 3 should return valid 2 byte
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 3, mb);
+  ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_EQ(static_cast<int>(*dest), 142);
+}
+
+TEST(LlvmLibcMBRToWC, TwoValidTwoBytes) {
+  const char ch[4] = {static_cast<char>(0xC2), static_cast<char>(0x8E),
+                      static_cast<char>(0xC7), static_cast<char>(0x8C)};
+  wchar_t dest[2];
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  // mbstate should reset after reading first one
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, ch, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_EQ(static_cast<int>(*dest), 142);
+  n = LIBC_NAMESPACE::mbrtowc(dest + 1, ch + 2, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 2);
+  ASSERT_EQ(static_cast<int>(*(dest + 1)), 460);
+}
+
+TEST(LlvmLibcMBRToWC, NullString) {
+  wchar_t dest[2] = {L'O', L'K'};
+  mbstate_t *mb;
+  LIBC_NAMESPACE::memset(&mb, 0, sizeof(mbstate_t));
+  // reading on nullptr should return 0
+  size_t n = LIBC_NAMESPACE::mbrtowc(dest, nullptr, 2, mb);
+  ASSERT_EQ(static_cast<int>(n), 0);
+  ASSERT_TRUE(dest[0] == L'O');
+  // reading a null terminator should return 0
+  const char *ch = "\0";
+  n = LIBC_NAMESPACE::mbrtowc(dest, ch, 1, mb);
+  ASSERT_EQ(static_cast<int>(n), 0);
+}
-- 
cgit v1.2.3


From f8ffb4e7cd94b661c3edd323f3dd85dc77892c16 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Fri, 20 Jun 2025 21:08:14 +0100
Subject: [VPlan] Simplify ExtractLastElement(Broadcast(A)) -> A.

Remove trivial ExtractLastElement VPInstructions.
---
 llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp                  | 7 +++++++
 .../LoopVectorize/first-order-recurrence-dead-instructions.ll      | 6 ++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index fbfc527ff7db..c0bdbb1f4f88 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -1164,6 +1164,13 @@ static void simplifyRecipe(VPRecipeBase &R, VPTypeAnalysis &TypeInfo) {
     return;
   }
 
+  if (match(Def,
+            m_VPInstruction<VPInstruction::ExtractLastElement>(
+                m_VPInstruction<VPInstruction::Broadcast>(m_VPValue(A))))) {
+    Def->replaceAllUsesWith(A);
+    return;
+  }
+
   VPInstruction *OpVPI;
   if (match(Def, m_VPInstruction<VPInstruction::ExtractLastElement>(
                      m_VPInstruction(OpVPI))) &&
diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
index d98cd45cb634..3bcb832b3fe3 100644
--- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
+++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-dead-instructions.ll
@@ -26,12 +26,10 @@ define i8 @recurrence_phi_with_same_incoming_values_after_simplifications(i8 %fo
 ; CHECK-NEXT:    [[TMP4:%.*]] = icmp eq i32 [[INDEX_NEXT]], -8
 ; CHECK-NEXT:    br i1 [[TMP4]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
 ; CHECK:       [[MIDDLE_BLOCK]]:
-; CHECK-NEXT:    [[TMP5:%.*]] = extractelement <4 x i8> [[BROADCAST_SPLAT]], i32 3
-; CHECK-NEXT:    [[VECTOR_RECUR_EXTRACT:%.*]] = extractelement <4 x i8> [[BROADCAST_SPLAT]], i32 3
 ; CHECK-NEXT:    br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]]
 ; CHECK:       [[SCALAR_PH]]:
 ; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i32 [ -7, %[[MIDDLE_BLOCK]] ], [ 1, %[[ENTRY]] ], [ 1, %[[VECTOR_SCEVCHECK]] ]
-; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[VECTOR_RECUR_EXTRACT]], %[[MIDDLE_BLOCK]] ], [ [[FOR_START]], %[[ENTRY]] ], [ [[FOR_START]], %[[VECTOR_SCEVCHECK]] ]
+; CHECK-NEXT:    [[SCALAR_RECUR_INIT:%.*]] = phi i8 [ [[FOR_START]], %[[MIDDLE_BLOCK]] ], [ [[FOR_START]], %[[ENTRY]] ], [ [[FOR_START]], %[[VECTOR_SCEVCHECK]] ]
 ; CHECK-NEXT:    br label %[[LOOP:.*]]
 ; CHECK:       [[LOOP]]:
 ; CHECK-NEXT:    [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
@@ -43,7 +41,7 @@ define i8 @recurrence_phi_with_same_incoming_values_after_simplifications(i8 %fo
 ; CHECK-NEXT:    [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 0
 ; CHECK-NEXT:    br i1 [[EC]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
 ; CHECK:       [[EXIT]]:
-; CHECK-NEXT:    [[FOR_NEXT_LCSSA:%.*]] = phi i8 [ [[FOR_NEXT]], %[[LOOP]] ], [ [[TMP5]], %[[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    [[FOR_NEXT_LCSSA:%.*]] = phi i8 [ [[FOR_NEXT]], %[[LOOP]] ], [ [[FOR_START]], %[[MIDDLE_BLOCK]] ]
 ; CHECK-NEXT:    ret i8 [[FOR_NEXT_LCSSA]]
 ;
 entry:
-- 
cgit v1.2.3


From 3b9795b3d3b249a5a3052a44f2c1ad7268ed34c6 Mon Sep 17 00:00:00 2001
From: Rodolfo Wottrich <rgwott@users.noreply.github.com>
Date: Fri, 20 Jun 2025 21:14:52 +0100
Subject: [AArch64] Add CodeGen support for scalar FEAT_CPA (#105669)

CPA stands for Checked Pointer Arithmetic and is part of the 2023 MTE
architecture extensions for A-profile.
The new CPA instructions perform regular pointer arithmetic (such as
base register + offset) but check for overflow in the most significant
bits of the result, enhancing security by detecting address tampering.

In this patch we intend to capture the semantics of pointer arithmetic
when it is not folded into loads/stores, then generate the appropriate
scalar CPA instructions. In order to preserve pointer arithmetic
semantics through the backend, we use the PTRADD SelectionDAG node type.

Use backend option `-aarch64-use-featcpa-codegen=true` to enable CPA
CodeGen (for a target with CPA enabled).

The story of this PR is that initially it introduced the PTRADD
SelectionDAG node and the respective visitPTRADD() function, adapted
from the CHERI/Morello LLVM tree. The original authors are
@davidchisnall, @jrtc27, @arichardson.
After a while, @ritter-x2a took the part of the code that was
target-independent and merged it separately in #140017. This PR thus
remains as the AArch64-part only.

Mode details about the CPA extension can be found at:

-
https://community.arm.com/arm-community-blogs/b/architectures-and-processors-blog/posts/arm-a-profile-architecture-developments-2023
- https://developer.arm.com/documentation/ddi0602/2023-09/ (e.g ADDPT
instruction)

This PR follows #79569.
It does not address vector FEAT_CPA instructions.
---
 llvm/lib/Target/AArch64/AArch64ISelLowering.cpp    |  13 +
 llvm/lib/Target/AArch64/AArch64ISelLowering.h      |   4 +
 llvm/lib/Target/AArch64/AArch64InstrFormats.td     |   2 +-
 llvm/lib/Target/AArch64/AArch64InstrInfo.td        |  15 +
 .../AArch64/GISel/AArch64InstructionSelector.cpp   |   8 +-
 llvm/test/CodeGen/AArch64/cpa-globalisel.ll        | 776 ++++++++++++++++++++
 llvm/test/CodeGen/AArch64/cpa-selectiondag.ll      | 796 +++++++++++++++++++++
 7 files changed, 1612 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/cpa-globalisel.ll
 create mode 100644 llvm/test/CodeGen/AArch64/cpa-selectiondag.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index e9d05710cbc4..a2c914c6e09c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -153,6 +153,14 @@ cl::opt<bool> EnableSVEGISel(
     cl::desc("Enable / disable SVE scalable vectors in Global ISel"),
     cl::init(false));
 
+// TODO: This option should be removed once we switch to always using PTRADD in
+// the SelectionDAG.
+static cl::opt<bool> UseFEATCPACodegen(
+    "aarch64-use-featcpa-codegen", cl::Hidden,
+    cl::desc("Generate ISD::PTRADD nodes for pointer arithmetic in "
+             "SelectionDAG for FEAT_CPA"),
+    cl::init(false));
+
 /// Value type used for condition codes.
 static const MVT MVT_CC = MVT::i32;
 
@@ -30480,3 +30488,8 @@ bool AArch64TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
 
   return TargetLowering::isTypeDesirableForOp(Opc, VT);
 }
+
+bool AArch64TargetLowering::shouldPreservePtrArith(const Function &F,
+                                                   EVT VT) const {
+  return Subtarget->hasCPA() && UseFEATCPACodegen;
+}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index e0b6c1b8c0ba..89f90ee2b770 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -539,6 +539,10 @@ public:
   /// True if stack clash protection is enabled for this functions.
   bool hasInlineStackProbe(const MachineFunction &MF) const override;
 
+  /// In AArch64, true if FEAT_CPA is present. Allows pointer arithmetic
+  /// semantics to be preserved for instruction selection.
+  bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override;
+
 private:
   /// Keep a pointer to the AArch64Subtarget around so that we can
   /// make the right decision when generating code for different targets.
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index 9078748c1483..ba7cbccc0bcd 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -13124,7 +13124,7 @@ def LSLImm3ShiftOperand : AsmOperandClass {
   let DiagnosticType = "AddSubLSLImm3ShiftLarge";
 }
 
-def lsl_imm3_shift_operand : Operand<i32> {
+def lsl_imm3_shift_operand : Operand<i64> {
   let PrintMethod = "printShifter";
   let ParserMatchClass = LSLImm3ShiftOperand;
 }
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index efe6cc1aa8ae..0f3f24f0853c 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -10741,6 +10741,21 @@ let Predicates = [HasCPA] in {
   // Scalar multiply-add/subtract
   def MADDPT : MulAccumCPA<0, "maddpt">;
   def MSUBPT : MulAccumCPA<1, "msubpt">;
+
+  def : Pat<(ptradd GPR64sp:$Rn, GPR64sp:$Rm),
+            (ADDPT_shift GPR64sp:$Rn, GPR64sp:$Rm, (i64 0))>;
+  def : Pat<(ptradd GPR64sp:$Rn, (shl GPR64sp:$Rm, (i64 imm0_7:$imm))),
+            (ADDPT_shift GPR64sp:$Rn, GPR64sp:$Rm,
+                         (i64 imm0_7:$imm))>;
+  def : Pat<(ptradd GPR64sp:$Rn, (ineg GPR64sp:$Rm)),
+            (SUBPT_shift GPR64sp:$Rn, GPR64sp:$Rm, (i64 0))>;
+  def : Pat<(ptradd GPR64sp:$Rn, (ineg (shl GPR64sp:$Rm, (i64 imm0_7:$imm)))),
+            (SUBPT_shift GPR64sp:$Rn, GPR64sp:$Rm,
+                         (i64 imm0_7:$imm))>;
+  def : Pat<(ptradd GPR64:$Ra, (mul GPR64:$Rn, GPR64:$Rm)),
+            (MADDPT GPR64:$Rn, GPR64:$Rm, GPR64:$Ra)>;
+  def : Pat<(ptradd GPR64:$Ra, (mul GPR64:$Rn, (ineg GPR64:$Rm))),
+            (MSUBPT GPR64:$Rn, GPR64:$Rm, GPR64:$Ra)>;
 }
 
 def round_v4fp32_to_v4bf16 :
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
index 51b42325ef84..5081cc4bba14 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64InstructionSelector.cpp
@@ -2197,8 +2197,14 @@ bool AArch64InstructionSelector::preISelLower(MachineInstr &I) {
     }
     return Changed;
   }
-  case TargetOpcode::G_PTR_ADD:
+  case TargetOpcode::G_PTR_ADD: {
+    // If Checked Pointer Arithmetic (FEAT_CPA) is present, preserve the pointer
+    // arithmetic semantics instead of falling back to regular arithmetic.
+    const auto &TL = STI.getTargetLowering();
+    if (TL->shouldPreservePtrArith(MF.getFunction(), EVT()))
+      return false;
     return convertPtrAddToAdd(I, MRI);
+  }
   case TargetOpcode::G_LOAD: {
     // For scalar loads of pointers, we try to convert the dest type from p0
     // to s64 so that our imported patterns can match. Like with the G_PTR_ADD
diff --git a/llvm/test/CodeGen/AArch64/cpa-globalisel.ll b/llvm/test/CodeGen/AArch64/cpa-globalisel.ll
new file mode 100644
index 000000000000..c9b48b9685df
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cpa-globalisel.ll
@@ -0,0 +1,776 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=+cpa -aarch64-use-featcpa-codegen=true -O0 -global-isel=1 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-CPA-O0
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=+cpa -aarch64-use-featcpa-codegen=true -O3 -global-isel=1 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-CPA-O3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=-cpa -O0 -global-isel=1 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-NOCPA-O0
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=-cpa -O3 -global-isel=1 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-NOCPA-O3
+
+%struct.my_type = type { i64, i64 }
+%struct.my_type2 = type { i64, i64, i64, i64, i64, i64 }
+
+@array = external dso_local global [10 x %struct.my_type], align 8
+@array2 = external dso_local global [10 x %struct.my_type2], align 8
+
+define void @addpt1(i64 %index, i64 %arg) {
+; CHECK-CPA-O0-LABEL: addpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    adrp x8, array
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x0, lsl #4
+; CHECK-CPA-O0-NEXT:    str x1, [x8, #8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    adrp x8, array
+; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O3-NEXT:    addpt x8, x8, x0, lsl #4
+; CHECK-CPA-O3-NEXT:    str x1, [x8, #8]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    adrp x8, array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x0, lsl #4
+; CHECK-NOCPA-O0-NEXT:    str x1, [x8, #8]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    adrp x8, array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, x0, lsl #4
+; CHECK-NOCPA-O3-NEXT:    str x1, [x8, #8]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %e2 = getelementptr inbounds %struct.my_type, ptr @array, i64 %index, i32 1
+  store i64 %arg, ptr %e2, align 8
+  ret void
+}
+
+define void @maddpt1(i32 %pos, ptr %val) {
+; CHECK-CPA-O0-LABEL: maddpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w0
+; CHECK-CPA-O0-NEXT:    sxtw x8, w8
+; CHECK-CPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    adrp x10, array2
+; CHECK-CPA-O0-NEXT:    add x10, x10, :lo12:array2
+; CHECK-CPA-O0-NEXT:    maddpt x0, x8, x9, x10
+; CHECK-CPA-O0-NEXT:    mov w8, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    mov w2, w8
+; CHECK-CPA-O0-NEXT:    b memcpy
+;
+; CHECK-CPA-O3-LABEL: maddpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-CPA-O3-NEXT:    sxtw x9, w0
+; CHECK-CPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    ldr q0, [x1]
+; CHECK-CPA-O3-NEXT:    adrp x10, array2
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:array2
+; CHECK-CPA-O3-NEXT:    maddpt x8, x9, x8, x10
+; CHECK-CPA-O3-NEXT:    str q0, [x8]
+; CHECK-CPA-O3-NEXT:    ldr q0, [x1, #16]
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #16]
+; CHECK-CPA-O3-NEXT:    ldr q0, [x1, #32]
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #32]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: maddpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O0-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O0-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    smaddl x0, w0, w8, x9
+; CHECK-NOCPA-O0-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    mov w2, w8
+; CHECK-NOCPA-O0-NEXT:    b memcpy
+;
+; CHECK-NOCPA-O3-LABEL: maddpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O3-NEXT:    smaddl x8, w0, w8, x9
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x1]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8]
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x1, #16]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #16]
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x1, #32]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #32]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idxprom = sext i32 %pos to i64
+  %arrayidx = getelementptr inbounds [10 x %struct.my_type2], ptr @array2, i64 0, i64 %idxprom
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 dereferenceable(48) %arrayidx, ptr align 8 dereferenceable(48) %val, i64 48, i1 false)
+  ret void
+}
+
+define void @msubpt1(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: msubpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w0
+; CHECK-CPA-O0-NEXT:    sxtw x10, w8
+; CHECK-CPA-O0-NEXT:    mov w8, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    mov w9, w8
+; CHECK-CPA-O0-NEXT:    mov w8, #288 // =0x120
+; CHECK-CPA-O0-NEXT:    mov w11, w8
+; CHECK-CPA-O0-NEXT:    adrp x8, array2
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:array2
+; CHECK-CPA-O0-NEXT:    addpt x11, x8, x11
+; CHECK-CPA-O0-NEXT:    msubpt x0, x9, x10, x11
+; CHECK-CPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    mov w2, w9
+; CHECK-CPA-O0-NEXT:    mov w9, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    addpt x1, x8, x9
+; CHECK-CPA-O0-NEXT:    b memcpy
+;
+; CHECK-CPA-O3-LABEL: msubpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-CPA-O3-NEXT:    sxtw x9, w0
+; CHECK-CPA-O3-NEXT:    adrp x10, array2+96
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:array2+96
+; CHECK-CPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    ldr q0, [x10]
+; CHECK-CPA-O3-NEXT:    msubpt x8, x8, x9, x10
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #192]
+; CHECK-CPA-O3-NEXT:    ldr q0, [x10, #16]
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #208]
+; CHECK-CPA-O3-NEXT:    ldr q0, [x10, #32]
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #224]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: msubpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-NOCPA-O0-NEXT:    mov w8, w0
+; CHECK-NOCPA-O0-NEXT:    sxtw x8, w8
+; CHECK-NOCPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-NOCPA-O0-NEXT:    mneg x10, x8, x9
+; CHECK-NOCPA-O0-NEXT:    adrp x8, array2
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:array2
+; CHECK-NOCPA-O0-NEXT:    add x9, x8, #288
+; CHECK-NOCPA-O0-NEXT:    add x0, x9, x10
+; CHECK-NOCPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    mov w2, w9
+; CHECK-NOCPA-O0-NEXT:    add x1, x8, #96
+; CHECK-NOCPA-O0-NEXT:    b memcpy
+;
+; CHECK-NOCPA-O3-LABEL: msubpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NOCPA-O3-NEXT:    sxtw x8, w0
+; CHECK-NOCPA-O3-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    mneg x8, x8, x9
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array2+96
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array2+96
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x9]
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #192]
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x9, #16]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #208]
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x9, #32]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #224]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idx.ext = sext i32 %index to i64
+  %idx.neg = sub nsw i64 0, %idx.ext
+  %add.ptr = getelementptr inbounds %struct.my_type2, ptr getelementptr inbounds ([10 x %struct.my_type2], ptr @array2, i64 0, i64 6), i64 %idx.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 dereferenceable(48) %add.ptr, ptr align 8 dereferenceable(48) getelementptr inbounds ([10 x %struct.my_type2], ptr @array2, i64 0, i64 2), i64 48, i1 false), !tbaa.struct !6
+  ret void
+}
+
+define void @subpt1(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: subpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    mov w8, #-16 // =0xfffffff0
+; CHECK-CPA-O0-NEXT:    smull x9, w0, w8
+; CHECK-CPA-O0-NEXT:    adrp x8, array
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O0-NEXT:    ldr q0, [x8, #32]
+; CHECK-CPA-O0-NEXT:    mov w10, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x10 killed $w10
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10
+; CHECK-CPA-O0-NEXT:    str q0, [x8, x9, lsl #4]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    mov w8, #-16 // =0xfffffff0
+; CHECK-CPA-O3-NEXT:    adrp x9, array+32
+; CHECK-CPA-O3-NEXT:    add x9, x9, :lo12:array+32
+; CHECK-CPA-O3-NEXT:    smull x8, w0, w8
+; CHECK-CPA-O3-NEXT:    ldr q0, [x9]
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8, lsl #4
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #64]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    mov w8, #-16 // =0xfffffff0
+; CHECK-NOCPA-O0-NEXT:    smull x9, w0, w8
+; CHECK-NOCPA-O0-NEXT:    adrp x8, array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x8, #32]
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, #96
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8, x9, lsl #4]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    mov w8, #-16 // =0xfffffff0
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array+32
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array+32
+; CHECK-NOCPA-O3-NEXT:    smull x8, w0, w8
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x9]
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8, lsl #4
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #64]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %conv = sext i32 %index to i64
+  %mul.neg = mul nsw i64 %conv, -16
+  %add.ptr = getelementptr inbounds %struct.my_type, ptr getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 6), i64 %mul.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %add.ptr, ptr noundef nonnull align 8 dereferenceable(16) getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 2), i64 16, i1 false), !tbaa.struct !6
+  ret void
+}
+
+define void @subpt2(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: subpt2:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    mov x8, xzr
+; CHECK-CPA-O0-NEXT:    subs x9, x8, w0, sxtw
+; CHECK-CPA-O0-NEXT:    adrp x8, array
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O0-NEXT:    ldr q0, [x8, #32]
+; CHECK-CPA-O0-NEXT:    mov w10, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x10 killed $w10
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10
+; CHECK-CPA-O0-NEXT:    str q0, [x8, x9, lsl #4]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt2:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    mov x8, xzr
+; CHECK-CPA-O3-NEXT:    adrp x9, array+32
+; CHECK-CPA-O3-NEXT:    add x9, x9, :lo12:array+32
+; CHECK-CPA-O3-NEXT:    sub x8, x8, w0, sxtw
+; CHECK-CPA-O3-NEXT:    ldr q0, [x9]
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8, lsl #4
+; CHECK-CPA-O3-NEXT:    str q0, [x8, #64]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt2:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    mov x8, xzr
+; CHECK-NOCPA-O0-NEXT:    subs x9, x8, w0, sxtw
+; CHECK-NOCPA-O0-NEXT:    adrp x8, array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x8, #32]
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, #96
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8, x9, lsl #4]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt2:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    mov x8, xzr
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array+32
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array+32
+; CHECK-NOCPA-O3-NEXT:    sub x8, x8, w0, sxtw
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x9]
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8, lsl #4
+; CHECK-NOCPA-O3-NEXT:    str q0, [x8, #64]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idx.ext = sext i32 %index to i64
+  %idx.neg = sub nsw i64 0, %idx.ext
+  %add.ptr = getelementptr inbounds %struct.my_type, ptr getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 6), i64 %idx.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %add.ptr, ptr noundef nonnull align 8 dereferenceable(16) getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 2), i64 16, i1 false), !tbaa.struct !11
+  ret void
+}
+
+define ptr @subpt3(ptr %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subpt3:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    mov x8, #-8 // =0xfffffffffffffff8
+; CHECK-CPA-O0-NEXT:    addpt x0, x0, x8
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt3:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    mov x8, #-8 // =0xfffffffffffffff8
+; CHECK-CPA-O3-NEXT:    addpt x0, x0, x8
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt3:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs x0, x0, #8
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt3:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub x0, x0, #8
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = getelementptr inbounds i64, ptr %ptr, i64 -1
+  ret ptr %incdec.ptr.i.i.i
+}
+
+define i64 @subi64(i64 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi64:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs x0, x0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi64:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub x0, x0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi64:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs x0, x0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi64:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub x0, x0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i64 %ptr, -1
+  ret i64 %incdec.ptr.i.i.i
+}
+
+define i32 @subi32(i32 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi32:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi32:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi32:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi32:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i32 %ptr, -1
+  ret i32 %incdec.ptr.i.i.i
+}
+
+define i16 @subi16(i16 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi16:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi16:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi16:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi16:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i16 %ptr, -1
+  ret i16 %incdec.ptr.i.i.i
+}
+
+define i64 @addi64(i64 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi64:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add x0, x0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi64:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add x0, x0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi64:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add x0, x0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi64:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add x0, x0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i64 %ptr, 1
+  ret i64 %incdec.ptr.i.i.i
+}
+
+define i32 @addi32(i32 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi32:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi32:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi32:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi32:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i32 %ptr, 1
+  ret i32 %incdec.ptr.i.i.i
+}
+
+define i16 @addi16(i16 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi16:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi16:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi16:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi16:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i16 %ptr, 1
+  ret i16 %incdec.ptr.i.i.i
+}
+
+define i64 @arith1(i64 noundef %0, i64 noundef %1, i64 noundef %2) {
+; CHECK-CPA-O0-LABEL: arith1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-CPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-CPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-CPA-O0-NEXT:    ldr x9, [sp, #16]
+; CHECK-CPA-O0-NEXT:    ldr x10, [sp, #8]
+; CHECK-CPA-O0-NEXT:    mul x9, x9, x10
+; CHECK-CPA-O0-NEXT:    add x0, x8, x9
+; CHECK-CPA-O0-NEXT:    add sp, sp, #32
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: arith1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O3-NEXT:    stp x2, x1, [sp, #8]
+; CHECK-CPA-O3-NEXT:    str x0, [sp, #24]
+; CHECK-CPA-O3-NEXT:    madd x0, x1, x2, x0
+; CHECK-CPA-O3-NEXT:    add sp, sp, #32
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: arith1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    ldr x9, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    ldr x10, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    mul x9, x9, x10
+; CHECK-NOCPA-O0-NEXT:    add x0, x8, x9
+; CHECK-NOCPA-O0-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: arith1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O3-NEXT:    stp x2, x1, [sp, #8]
+; CHECK-NOCPA-O3-NEXT:    str x0, [sp, #24]
+; CHECK-NOCPA-O3-NEXT:    madd x0, x1, x2, x0
+; CHECK-NOCPA-O3-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %4 = alloca i64, align 8
+  %5 = alloca i64, align 8
+  %6 = alloca i64, align 8
+  store i64 %0, ptr %4, align 8
+  store i64 %1, ptr %5, align 8
+  store i64 %2, ptr %6, align 8
+  %7 = load i64, ptr %4, align 8
+  %8 = load i64, ptr %5, align 8
+  %9 = load i64, ptr %6, align 8
+  %10 = mul nsw i64 %8, %9
+  %11 = add nsw i64 %7, %10
+  ret i64 %11
+}
+
+define i64 @arith2(ptr noundef %0, i64 noundef %1, i64 noundef %2, i32 noundef %3) {
+; CHECK-CPA-O0-LABEL: arith2:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-CPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-CPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O0-NEXT:    str w3, [sp, #4]
+; CHECK-CPA-O0-NEXT:    ldr x10, [sp, #24]
+; CHECK-CPA-O0-NEXT:    ldrsw x8, [sp, #4]
+; CHECK-CPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    maddpt x8, x8, x9, x10
+; CHECK-CPA-O0-NEXT:    ldr x8, [x8, #24]
+; CHECK-CPA-O0-NEXT:    ldr x10, [sp, #16]
+; CHECK-CPA-O0-NEXT:    ldr x9, [sp, #8]
+; CHECK-CPA-O0-NEXT:    mul x10, x10, x9
+; CHECK-CPA-O0-NEXT:    add x8, x8, x10
+; CHECK-CPA-O0-NEXT:    subs x0, x8, x9
+; CHECK-CPA-O0-NEXT:    add sp, sp, #32
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: arith2:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O3-NEXT:    str w3, [sp, #4]
+; CHECK-CPA-O3-NEXT:    mov w10, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    ldrsw x9, [sp, #4]
+; CHECK-CPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-CPA-O3-NEXT:    maddpt x8, x9, x10, x0
+; CHECK-CPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O3-NEXT:    ldr x8, [x8, #24]
+; CHECK-CPA-O3-NEXT:    madd x8, x1, x2, x8
+; CHECK-CPA-O3-NEXT:    sub x0, x8, x2
+; CHECK-CPA-O3-NEXT:    add sp, sp, #32
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: arith2:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    str w3, [sp, #4]
+; CHECK-NOCPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    ldrsw x9, [sp, #4]
+; CHECK-NOCPA-O0-NEXT:    mov w10, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    // kill: def $x10 killed $w10
+; CHECK-NOCPA-O0-NEXT:    mul x9, x9, x10
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x9
+; CHECK-NOCPA-O0-NEXT:    ldr x8, [x8, #24]
+; CHECK-NOCPA-O0-NEXT:    ldr x10, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    ldr x9, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    mul x10, x10, x9
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x10
+; CHECK-NOCPA-O0-NEXT:    subs x0, x8, x9
+; CHECK-NOCPA-O0-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: arith2:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O3-NEXT:    str w3, [sp, #4]
+; CHECK-NOCPA-O3-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    ldrsw x8, [sp, #4]
+; CHECK-NOCPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-NOCPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O3-NEXT:    madd x8, x8, x9, x0
+; CHECK-NOCPA-O3-NEXT:    ldr x8, [x8, #24]
+; CHECK-NOCPA-O3-NEXT:    madd x8, x1, x2, x8
+; CHECK-NOCPA-O3-NEXT:    sub x0, x8, x2
+; CHECK-NOCPA-O3-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %5 = alloca ptr, align 8
+  %6 = alloca i64, align 8
+  %7 = alloca i64, align 8
+  %8 = alloca i32, align 4
+  store ptr %0, ptr %5, align 8
+  store i64 %1, ptr %6, align 8
+  store i64 %2, ptr %7, align 8
+  store i32 %3, ptr %8, align 4
+  %9 = load ptr, ptr %5, align 8
+  %10 = load i32, ptr %8, align 4
+  %11 = sext i32 %10 to i64
+  %12 = getelementptr inbounds %struct.my_type2, ptr %9, i64 %11
+  %13 = getelementptr inbounds %struct.my_type2, ptr %12, i32 0, i32 3
+  %14 = load i64, ptr %13, align 8
+  %15 = load i64, ptr %6, align 8
+  %16 = load i64, ptr %7, align 8
+  %17 = mul nsw i64 %15, %16
+  %18 = add nsw i64 %14, %17
+  %19 = sub nsw i64 %18, %16
+  ret i64 %19
+}
+
+@a = hidden global [2 x [1 x [2 x i8]]] [[1 x [2 x i8]] [[2 x i8] c"\01\01"], [1 x [2 x i8]] [[2 x i8] c"\01\01"]], align 1
+@b = hidden global i16 0, align 2
+@.str = private unnamed_addr constant [7 x i8] c"hello\0A\00", align 1
+
+define hidden void @multidim() {
+; CHECK-CPA-O0-LABEL: multidim:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-CPA-O0-NEXT:    .cfi_offset w30, -16
+; CHECK-CPA-O0-NEXT:    adrp x8, b
+; CHECK-CPA-O0-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-CPA-O0-NEXT:    mov w10, w9
+; CHECK-CPA-O0-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-CPA-O0-NEXT:    add w9, w8, #1
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w9
+; CHECK-CPA-O0-NEXT:    sxtw x9, w8
+; CHECK-CPA-O0-NEXT:    mov w8, #2 // =0x2
+; CHECK-CPA-O0-NEXT:    mov w11, w8
+; CHECK-CPA-O0-NEXT:    adrp x8, a
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:a
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x11
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #1
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x9
+; CHECK-CPA-O0-NEXT:    ldrb w8, [x8]
+; CHECK-CPA-O0-NEXT:    cbz w8, .LBB14_2
+; CHECK-CPA-O0-NEXT:    b .LBB14_1
+; CHECK-CPA-O0-NEXT:  .LBB14_1:
+; CHECK-CPA-O0-NEXT:    adrp x0, .L.str
+; CHECK-CPA-O0-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-CPA-O0-NEXT:    bl printf
+; CHECK-CPA-O0-NEXT:    b .LBB14_2
+; CHECK-CPA-O0-NEXT:  .LBB14_2:
+; CHECK-CPA-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: multidim:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    adrp x8, b
+; CHECK-CPA-O3-NEXT:    adrp x10, a+2
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:a+2
+; CHECK-CPA-O3-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-CPA-O3-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9, lsl #1
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O3-NEXT:    ldrb w8, [x8, #1]
+; CHECK-CPA-O3-NEXT:    cbz w8, .LBB14_2
+; CHECK-CPA-O3-NEXT:  // %bb.1:
+; CHECK-CPA-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-CPA-O3-NEXT:    .cfi_offset w30, -16
+; CHECK-CPA-O3-NEXT:    adrp x0, .L.str
+; CHECK-CPA-O3-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-CPA-O3-NEXT:    bl printf
+; CHECK-CPA-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CPA-O3-NEXT:  .LBB14_2:
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: multidim:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOCPA-O0-NEXT:    .cfi_offset w30, -16
+; CHECK-NOCPA-O0-NEXT:    adrp x8, b
+; CHECK-NOCPA-O0-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-NOCPA-O0-NEXT:    mov w10, w9
+; CHECK-NOCPA-O0-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-NOCPA-O0-NEXT:    add w9, w8, #1
+; CHECK-NOCPA-O0-NEXT:    adrp x8, a
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:a
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, #2
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x10, lsl #1
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, w9, sxtw
+; CHECK-NOCPA-O0-NEXT:    ldrb w8, [x8]
+; CHECK-NOCPA-O0-NEXT:    cbz w8, .LBB14_2
+; CHECK-NOCPA-O0-NEXT:    b .LBB14_1
+; CHECK-NOCPA-O0-NEXT:  .LBB14_1:
+; CHECK-NOCPA-O0-NEXT:    adrp x0, .L.str
+; CHECK-NOCPA-O0-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-NOCPA-O0-NEXT:    bl printf
+; CHECK-NOCPA-O0-NEXT:    b .LBB14_2
+; CHECK-NOCPA-O0-NEXT:  .LBB14_2:
+; CHECK-NOCPA-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: multidim:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    adrp x8, b
+; CHECK-NOCPA-O3-NEXT:    adrp x10, a+2
+; CHECK-NOCPA-O3-NEXT:    add x10, x10, :lo12:a+2
+; CHECK-NOCPA-O3-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-NOCPA-O3-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-NOCPA-O3-NEXT:    add x9, x10, x9, lsl #1
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8
+; CHECK-NOCPA-O3-NEXT:    ldrb w8, [x8, #1]
+; CHECK-NOCPA-O3-NEXT:    cbz w8, .LBB14_2
+; CHECK-NOCPA-O3-NEXT:  // %bb.1:
+; CHECK-NOCPA-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOCPA-O3-NEXT:    .cfi_offset w30, -16
+; CHECK-NOCPA-O3-NEXT:    adrp x0, .L.str
+; CHECK-NOCPA-O3-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-NOCPA-O3-NEXT:    bl printf
+; CHECK-NOCPA-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOCPA-O3-NEXT:  .LBB14_2:
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %1 = load i16, ptr @b, align 2
+  %2 = zext i16 %1 to i64
+  %3 = getelementptr inbounds nuw [1 x [2 x i8]], ptr getelementptr inbounds ([2 x [1 x [2 x i8]]], ptr @a, i64 0, i64 1), i64 0, i64 %2
+  %4 = load i16, ptr @b, align 2
+  %5 = zext i16 %4 to i32
+  %6 = add nsw i32 %5, 1
+  %7 = sext i32 %6 to i64
+  %8 = getelementptr inbounds [2 x i8], ptr %3, i64 0, i64 %7
+  %9 = load i8, ptr %8, align 1
+  %10 = icmp ne i8 %9, 0
+  br i1 %10, label %11, label %13
+
+11:
+  %12 = call i32 (ptr, ...) @printf(ptr noundef @.str)
+  br label %13
+
+13:
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+declare i32 @printf(ptr noundef, ...) #1
+
+!6 = !{i64 0, i64 8, !7, i64 8, i64 8, !7, i64 16, i64 8, !7, i64 24, i64 8, !7, i64 32, i64 8, !7, i64 40, i64 8, !7}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"long", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C++ TBAA"}
+!11 = !{i64 0, i64 8, !7, i64 8, i64 8, !7}
diff --git a/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll b/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll
new file mode 100644
index 000000000000..69fd6c4de78b
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cpa-selectiondag.ll
@@ -0,0 +1,796 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=+cpa -aarch64-use-featcpa-codegen=true -O0 -global-isel=0 -fast-isel=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-CPA-O0
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=+cpa -aarch64-use-featcpa-codegen=true -O3 -global-isel=0 -fast-isel=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-CPA-O3
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=-cpa -O0 -global-isel=0 -fast-isel=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-NOCPA-O0
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs --mattr=-cpa -O3 -global-isel=0 -fast-isel=0 %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK-NOCPA-O3
+
+%struct.my_type = type { i64, i64 }
+%struct.my_type2 = type { i64, i64, i64, i64, i64, i64 }
+
+@array = external dso_local global [10 x %struct.my_type], align 8
+@array2 = external dso_local global [10 x %struct.my_type2], align 8
+
+define void @addpt1(i64 %index, i64 %arg) {
+; CHECK-CPA-O0-LABEL: addpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    adrp x8, array
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x0, lsl #4
+; CHECK-CPA-O0-NEXT:    str x1, [x8, #8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    adrp x8, array
+; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O3-NEXT:    addpt x8, x8, x0, lsl #4
+; CHECK-CPA-O3-NEXT:    str x1, [x8, #8]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    adrp x8, array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x0, lsl #4
+; CHECK-NOCPA-O0-NEXT:    str x1, [x8, #8]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    adrp x8, array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, x0, lsl #4
+; CHECK-NOCPA-O3-NEXT:    str x1, [x8, #8]
+; CHECK-NOCPA-O3-NEXT:    ret
+
+entry:
+  %e2 = getelementptr inbounds %struct.my_type, ptr @array, i64 %index, i32 1
+  store i64 %arg, ptr %e2, align 8
+  ret void
+}
+
+define void @maddpt1(i32 %pos, ptr %val) {
+; CHECK-CPA-O0-LABEL: maddpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w0
+; CHECK-CPA-O0-NEXT:    sxtw x8, w8
+; CHECK-CPA-O0-NEXT:    adrp x10, array2
+; CHECK-CPA-O0-NEXT:    add x10, x10, :lo12:array2
+; CHECK-CPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    maddpt x8, x8, x9, x10
+; CHECK-CPA-O0-NEXT:    ldr q0, [x1]
+; CHECK-CPA-O0-NEXT:    ldr q1, [x1, #16]
+; CHECK-CPA-O0-NEXT:    ldr q2, [x1, #32]
+; CHECK-CPA-O0-NEXT:    str q2, [x8, #32]
+; CHECK-CPA-O0-NEXT:    str q1, [x8, #16]
+; CHECK-CPA-O0-NEXT:    str q0, [x8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: maddpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-CPA-O3-NEXT:    sxtw x9, w0
+; CHECK-CPA-O3-NEXT:    ldp q1, q0, [x1, #16]
+; CHECK-CPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    adrp x10, array2
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:array2
+; CHECK-CPA-O3-NEXT:    ldr q2, [x1]
+; CHECK-CPA-O3-NEXT:    maddpt x8, x9, x8, x10
+; CHECK-CPA-O3-NEXT:    stp q1, q0, [x8, #16]
+; CHECK-CPA-O3-NEXT:    str q2, [x8]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: maddpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O0-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O0-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    smaddl x8, w0, w8, x9
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x1]
+; CHECK-NOCPA-O0-NEXT:    ldr q1, [x1, #16]
+; CHECK-NOCPA-O0-NEXT:    ldr q2, [x1, #32]
+; CHECK-NOCPA-O0-NEXT:    str q2, [x8, #32]
+; CHECK-NOCPA-O0-NEXT:    str q1, [x8, #16]
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: maddpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O3-NEXT:    smaddl x8, w0, w8, x9
+; CHECK-NOCPA-O3-NEXT:    ldp q1, q0, [x1, #16]
+; CHECK-NOCPA-O3-NEXT:    ldr q2, [x1]
+; CHECK-NOCPA-O3-NEXT:    stp q1, q0, [x8, #16]
+; CHECK-NOCPA-O3-NEXT:    str q2, [x8]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idxprom = sext i32 %pos to i64
+  %arrayidx = getelementptr inbounds [10 x %struct.my_type2], ptr @array2, i64 0, i64 %idxprom
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 dereferenceable(48) %arrayidx, ptr align 8 dereferenceable(48) %val, i64 48, i1 false)
+  ret void
+}
+
+define void @msubpt1(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: msubpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-CPA-O0-NEXT:    mov w8, w0
+; CHECK-CPA-O0-NEXT:    sxtw x9, w8
+; CHECK-CPA-O0-NEXT:    mov x8, xzr
+; CHECK-CPA-O0-NEXT:    subs x8, x8, x9
+; CHECK-CPA-O0-NEXT:    lsl x8, x8, #1
+; CHECK-CPA-O0-NEXT:    subs x10, x8, x9
+; CHECK-CPA-O0-NEXT:    adrp x9, array2
+; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array2
+; CHECK-CPA-O0-NEXT:    mov w8, #288 // =0x120
+; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
+; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #4
+; CHECK-CPA-O0-NEXT:    mov w10, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x10 killed $w10
+; CHECK-CPA-O0-NEXT:    addpt x10, x9, x10
+; CHECK-CPA-O0-NEXT:    ldr q1, [x10, #16]
+; CHECK-CPA-O0-NEXT:    ldr q2, [x10, #32]
+; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #96]
+; CHECK-CPA-O0-NEXT:    str q2, [x8, #32]
+; CHECK-CPA-O0-NEXT:    str q1, [x8, #16]
+; CHECK-CPA-O0-NEXT:    str q0, [x8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: msubpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-CPA-O3-NEXT:    sxtw x9, w0
+; CHECK-CPA-O3-NEXT:    adrp x8, array2
+; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array2
+; CHECK-CPA-O3-NEXT:    mov w11, #96 // =0x60
+; CHECK-CPA-O3-NEXT:    mov w12, #288 // =0x120
+; CHECK-CPA-O3-NEXT:    ldr q2, [x8, #96]
+; CHECK-CPA-O3-NEXT:    neg x10, x9
+; CHECK-CPA-O3-NEXT:    addpt x11, x8, x11
+; CHECK-CPA-O3-NEXT:    lsl x10, x10, #1
+; CHECK-CPA-O3-NEXT:    ldp q1, q0, [x11, #16]
+; CHECK-CPA-O3-NEXT:    sub x9, x10, x9
+; CHECK-CPA-O3-NEXT:    addpt x10, x8, x12
+; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9, lsl #4
+; CHECK-CPA-O3-NEXT:    stp q1, q0, [x9, #16]
+; CHECK-CPA-O3-NEXT:    str q2, [x9]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: msubpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-NOCPA-O0-NEXT:    mov w8, w0
+; CHECK-NOCPA-O0-NEXT:    sxtw x8, w8
+; CHECK-NOCPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-NOCPA-O0-NEXT:    mneg x8, x8, x9
+; CHECK-NOCPA-O0-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O0-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O0-NEXT:    add x8, x9, x8
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x9, #96]
+; CHECK-NOCPA-O0-NEXT:    ldr q1, [x9, #112]
+; CHECK-NOCPA-O0-NEXT:    ldr q2, [x9, #128]
+; CHECK-NOCPA-O0-NEXT:    str q2, [x8, #320]
+; CHECK-NOCPA-O0-NEXT:    str q1, [x8, #304]
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8, #288]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: msubpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NOCPA-O3-NEXT:    sxtw x8, w0
+; CHECK-NOCPA-O3-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    mneg x8, x8, x9
+; CHECK-NOCPA-O3-NEXT:    adrp x9, array2
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:array2
+; CHECK-NOCPA-O3-NEXT:    ldp q1, q0, [x9, #112]
+; CHECK-NOCPA-O3-NEXT:    ldr q2, [x9, #96]
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8
+; CHECK-NOCPA-O3-NEXT:    stp q1, q0, [x8, #304]
+; CHECK-NOCPA-O3-NEXT:    str q2, [x8, #288]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idx.ext = sext i32 %index to i64
+  %idx.neg = sub nsw i64 0, %idx.ext
+  %add.ptr = getelementptr inbounds %struct.my_type2, ptr getelementptr inbounds ([10 x %struct.my_type2], ptr @array2, i64 0, i64 6), i64 %idx.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr align 8 dereferenceable(48) %add.ptr, ptr align 8 dereferenceable(48) getelementptr inbounds ([10 x %struct.my_type2], ptr @array2, i64 0, i64 2), i64 48, i1 false), !tbaa.struct !6
+  ret void
+}
+
+define void @subpt1(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: subpt1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    adrp x9, array
+; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
+; CHECK-CPA-O0-NEXT:    mov w8, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
+; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O0-NEXT:    // implicit-def: $x10
+; CHECK-CPA-O0-NEXT:    mov w10, w0
+; CHECK-CPA-O0-NEXT:    sbfiz x10, x10, #8, #32
+; CHECK-CPA-O0-NEXT:    subpt x8, x8, x10
+; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #32]
+; CHECK-CPA-O0-NEXT:    str q0, [x8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-CPA-O3-NEXT:    adrp x8, array
+; CHECK-CPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-CPA-O3-NEXT:    mov w9, #96 // =0x60
+; CHECK-CPA-O3-NEXT:    sbfiz x10, x0, #8, #32
+; CHECK-CPA-O3-NEXT:    addpt x9, x8, x9
+; CHECK-CPA-O3-NEXT:    ldr q0, [x8, #32]
+; CHECK-CPA-O3-NEXT:    subpt x8, x9, x10
+; CHECK-CPA-O3-NEXT:    str q0, [x8]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    // implicit-def: $x8
+; CHECK-NOCPA-O0-NEXT:    mov w8, w0
+; CHECK-NOCPA-O0-NEXT:    sxtw x8, w8
+; CHECK-NOCPA-O0-NEXT:    adrp x9, array
+; CHECK-NOCPA-O0-NEXT:    add x9, x9, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    subs x8, x9, x8, lsl #8
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x9, #32]
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8, #96]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NOCPA-O3-NEXT:    sxtw x9, w0
+; CHECK-NOCPA-O3-NEXT:    adrp x8, array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x8, #32]
+; CHECK-NOCPA-O3-NEXT:    sub x9, x8, x9, lsl #8
+; CHECK-NOCPA-O3-NEXT:    str q0, [x9, #96]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %conv = sext i32 %index to i64
+  %mul.neg = mul nsw i64 %conv, -16
+  %add.ptr = getelementptr inbounds %struct.my_type, ptr getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 6), i64 %mul.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %add.ptr, ptr noundef nonnull align 8 dereferenceable(16) getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 2), i64 16, i1 false), !tbaa.struct !6
+  ret void
+}
+
+define void @subpt2(i32 %index, i32 %elem) {
+; CHECK-CPA-O0-LABEL: subpt2:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    mov x8, xzr
+; CHECK-CPA-O0-NEXT:    subs x10, x8, w0, sxtw
+; CHECK-CPA-O0-NEXT:    adrp x9, array
+; CHECK-CPA-O0-NEXT:    add x9, x9, :lo12:array
+; CHECK-CPA-O0-NEXT:    mov w8, #96 // =0x60
+; CHECK-CPA-O0-NEXT:    // kill: def $x8 killed $w8
+; CHECK-CPA-O0-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #4
+; CHECK-CPA-O0-NEXT:    ldr q0, [x9, #32]
+; CHECK-CPA-O0-NEXT:    str q0, [x8]
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt2:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    mov x8, xzr
+; CHECK-CPA-O3-NEXT:    mov w9, #96 // =0x60
+; CHECK-CPA-O3-NEXT:    adrp x10, array
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:array
+; CHECK-CPA-O3-NEXT:    sub x8, x8, w0, sxtw
+; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9
+; CHECK-CPA-O3-NEXT:    ldr q0, [x10, #32]
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8, lsl #4
+; CHECK-CPA-O3-NEXT:    str q0, [x8]
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt2:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    adrp x9, array
+; CHECK-NOCPA-O0-NEXT:    add x9, x9, :lo12:array
+; CHECK-NOCPA-O0-NEXT:    subs x8, x9, w0, sxtw #4
+; CHECK-NOCPA-O0-NEXT:    ldr q0, [x9, #32]
+; CHECK-NOCPA-O0-NEXT:    str q0, [x8, #96]
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt2:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    adrp x8, array
+; CHECK-NOCPA-O3-NEXT:    add x8, x8, :lo12:array
+; CHECK-NOCPA-O3-NEXT:    sub x9, x8, w0, sxtw #4
+; CHECK-NOCPA-O3-NEXT:    ldr q0, [x8, #32]
+; CHECK-NOCPA-O3-NEXT:    str q0, [x9, #96]
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %idx.ext = sext i32 %index to i64
+  %idx.neg = sub nsw i64 0, %idx.ext
+  %add.ptr = getelementptr inbounds %struct.my_type, ptr getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 6), i64 %idx.neg
+  tail call void @llvm.memcpy.p0.p0.i64(ptr noundef nonnull align 8 dereferenceable(16) %add.ptr, ptr noundef nonnull align 8 dereferenceable(16) getelementptr inbounds ([10 x %struct.my_type], ptr @array, i64 0, i64 2), i64 16, i1 false), !tbaa.struct !11
+  ret void
+}
+
+define ptr @subpt3(ptr %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subpt3:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    mov x8, #-8 // =0xfffffffffffffff8
+; CHECK-CPA-O0-NEXT:    addpt x0, x0, x8
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subpt3:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    mov x8, #-8 // =0xfffffffffffffff8
+; CHECK-CPA-O3-NEXT:    addpt x0, x0, x8
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subpt3:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs x0, x0, #8
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subpt3:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub x0, x0, #8
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = getelementptr inbounds i64, ptr %ptr, i64 -1
+  ret ptr %incdec.ptr.i.i.i
+}
+
+define i64 @subi64(i64 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi64:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs x0, x0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi64:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub x0, x0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi64:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs x0, x0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi64:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub x0, x0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i64 %ptr, -1
+  ret i64 %incdec.ptr.i.i.i
+}
+
+define i32 @subi32(i32 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi32:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi32:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi32:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi32:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i32 %ptr, -1
+  ret i32 %incdec.ptr.i.i.i
+}
+
+define i16 @subi16(i16 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: subi16:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: subi16:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: subi16:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    subs w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: subi16:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i16 %ptr, -1
+  ret i16 %incdec.ptr.i.i.i
+}
+
+define i64 @addi64(i64 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi64:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add x0, x0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi64:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add x0, x0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi64:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add x0, x0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi64:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add x0, x0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i64 %ptr, 1
+  ret i64 %incdec.ptr.i.i.i
+}
+
+define i32 @addi32(i32 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi32:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi32:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi32:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi32:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i32 %ptr, 1
+  ret i32 %incdec.ptr.i.i.i
+}
+
+define i16 @addi16(i16 %ptr, i32 %index) {
+; CHECK-CPA-O0-LABEL: addi16:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    add w0, w0, #1
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: addi16:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    add w0, w0, #1
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: addi16:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: addi16:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    add w0, w0, #1
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %incdec.ptr.i.i.i = add i16 %ptr, 1
+  ret i16 %incdec.ptr.i.i.i
+}
+
+define i64 @arith1(i64 noundef %0, i64 noundef %1, i64 noundef %2) {
+; CHECK-CPA-O0-LABEL: arith1:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-CPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-CPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-CPA-O0-NEXT:    ldr x9, [sp, #16]
+; CHECK-CPA-O0-NEXT:    ldr x10, [sp, #8]
+; CHECK-CPA-O0-NEXT:    mul x9, x9, x10
+; CHECK-CPA-O0-NEXT:    add x0, x8, x9
+; CHECK-CPA-O0-NEXT:    add sp, sp, #32
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: arith1:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O3-NEXT:    madd x8, x1, x2, x0
+; CHECK-CPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-CPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O3-NEXT:    mov x0, x8
+; CHECK-CPA-O3-NEXT:    add sp, sp, #32
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: arith1:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    ldr x9, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    ldr x10, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    mul x9, x9, x10
+; CHECK-NOCPA-O0-NEXT:    add x0, x8, x9
+; CHECK-NOCPA-O0-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: arith1:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O3-NEXT:    madd x8, x1, x2, x0
+; CHECK-NOCPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-NOCPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O3-NEXT:    mov x0, x8
+; CHECK-NOCPA-O3-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %4 = alloca i64, align 8
+  %5 = alloca i64, align 8
+  %6 = alloca i64, align 8
+  store i64 %0, ptr %4, align 8
+  store i64 %1, ptr %5, align 8
+  store i64 %2, ptr %6, align 8
+  %7 = load i64, ptr %4, align 8
+  %8 = load i64, ptr %5, align 8
+  %9 = load i64, ptr %6, align 8
+  %10 = mul nsw i64 %8, %9
+  %11 = add nsw i64 %7, %10
+  ret i64 %11
+}
+
+define i64 @arith2(ptr noundef %0, i64 noundef %1, i64 noundef %2, i32 noundef %3) {
+; CHECK-CPA-O0-LABEL: arith2:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-CPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-CPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O0-NEXT:    str w3, [sp, #4]
+; CHECK-CPA-O0-NEXT:    ldr x8, [sp, #24]
+; CHECK-CPA-O0-NEXT:    ldrsw x9, [sp, #4]
+; CHECK-CPA-O0-NEXT:    add x9, x9, x9, lsl #1
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x9, lsl #4
+; CHECK-CPA-O0-NEXT:    ldr x8, [x8, #24]
+; CHECK-CPA-O0-NEXT:    ldr x10, [sp, #16]
+; CHECK-CPA-O0-NEXT:    ldr x9, [sp, #8]
+; CHECK-CPA-O0-NEXT:    mul x10, x10, x9
+; CHECK-CPA-O0-NEXT:    add x8, x8, x10
+; CHECK-CPA-O0-NEXT:    subs x0, x8, x9
+; CHECK-CPA-O0-NEXT:    add sp, sp, #32
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: arith2:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-CPA-O3-NEXT:    // kill: def $w3 killed $w3 def $x3
+; CHECK-CPA-O3-NEXT:    sxtw x8, w3
+; CHECK-CPA-O3-NEXT:    mov w9, #48 // =0x30
+; CHECK-CPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-CPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-CPA-O3-NEXT:    maddpt x8, x8, x9, x0
+; CHECK-CPA-O3-NEXT:    str w3, [sp, #4]
+; CHECK-CPA-O3-NEXT:    ldr x8, [x8, #24]
+; CHECK-CPA-O3-NEXT:    madd x8, x1, x2, x8
+; CHECK-CPA-O3-NEXT:    sub x0, x8, x2
+; CHECK-CPA-O3-NEXT:    add sp, sp, #32
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: arith2:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O0-NEXT:    str x0, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    str x1, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    str w3, [sp, #4]
+; CHECK-NOCPA-O0-NEXT:    ldr x10, [sp, #24]
+; CHECK-NOCPA-O0-NEXT:    ldrsw x8, [sp, #4]
+; CHECK-NOCPA-O0-NEXT:    mov w9, #48 // =0x30
+; CHECK-NOCPA-O0-NEXT:    mov w0, w9
+; CHECK-NOCPA-O0-NEXT:    mov w9, w0
+; CHECK-NOCPA-O0-NEXT:    // kill: def $w8 killed $w8 killed $x8
+; CHECK-NOCPA-O0-NEXT:    smaddl x8, w8, w9, x10
+; CHECK-NOCPA-O0-NEXT:    ldr x8, [x8, #24]
+; CHECK-NOCPA-O0-NEXT:    ldr x10, [sp, #16]
+; CHECK-NOCPA-O0-NEXT:    ldr x9, [sp, #8]
+; CHECK-NOCPA-O0-NEXT:    mul x10, x10, x9
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x10
+; CHECK-NOCPA-O0-NEXT:    subs x0, x8, x9
+; CHECK-NOCPA-O0-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: arith2:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    sub sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 32
+; CHECK-NOCPA-O3-NEXT:    mov w8, #48 // =0x30
+; CHECK-NOCPA-O3-NEXT:    stp x1, x0, [sp, #16]
+; CHECK-NOCPA-O3-NEXT:    smaddl x8, w3, w8, x0
+; CHECK-NOCPA-O3-NEXT:    str x2, [sp, #8]
+; CHECK-NOCPA-O3-NEXT:    str w3, [sp, #4]
+; CHECK-NOCPA-O3-NEXT:    ldr x8, [x8, #24]
+; CHECK-NOCPA-O3-NEXT:    madd x8, x1, x2, x8
+; CHECK-NOCPA-O3-NEXT:    sub x0, x8, x2
+; CHECK-NOCPA-O3-NEXT:    add sp, sp, #32
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %5 = alloca ptr, align 8
+  %6 = alloca i64, align 8
+  %7 = alloca i64, align 8
+  %8 = alloca i32, align 4
+  store ptr %0, ptr %5, align 8
+  store i64 %1, ptr %6, align 8
+  store i64 %2, ptr %7, align 8
+  store i32 %3, ptr %8, align 4
+  %9 = load ptr, ptr %5, align 8
+  %10 = load i32, ptr %8, align 4
+  %11 = sext i32 %10 to i64
+  %12 = getelementptr inbounds %struct.my_type2, ptr %9, i64 %11
+  %13 = getelementptr inbounds %struct.my_type2, ptr %12, i32 0, i32 3
+  %14 = load i64, ptr %13, align 8
+  %15 = load i64, ptr %6, align 8
+  %16 = load i64, ptr %7, align 8
+  %17 = mul nsw i64 %15, %16
+  %18 = add nsw i64 %14, %17
+  %19 = sub nsw i64 %18, %16
+  ret i64 %19
+}
+
+@a = hidden global [2 x [1 x [2 x i8]]] [[1 x [2 x i8]] [[2 x i8] c"\01\01"], [1 x [2 x i8]] [[2 x i8] c"\01\01"]], align 1
+@b = hidden global i16 0, align 2
+@.str = private unnamed_addr constant [7 x i8] c"hello\0A\00", align 1
+
+define hidden void @multidim() {
+; CHECK-CPA-O0-LABEL: multidim:
+; CHECK-CPA-O0:       // %bb.0: // %entry
+; CHECK-CPA-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CPA-O0-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-CPA-O0-NEXT:    .cfi_offset w30, -16
+; CHECK-CPA-O0-NEXT:    adrp x8, b
+; CHECK-CPA-O0-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-CPA-O0-NEXT:    mov w8, w9
+; CHECK-CPA-O0-NEXT:    mov w10, w8
+; CHECK-CPA-O0-NEXT:    adrp x8, a
+; CHECK-CPA-O0-NEXT:    add x8, x8, :lo12:a
+; CHECK-CPA-O0-NEXT:    mov w11, #2 // =0x2
+; CHECK-CPA-O0-NEXT:    // kill: def $x11 killed $w11
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x11
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x10, lsl #1
+; CHECK-CPA-O0-NEXT:    add w9, w9, #1
+; CHECK-CPA-O0-NEXT:    mov w9, w9
+; CHECK-CPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-CPA-O0-NEXT:    addpt x8, x8, x9
+; CHECK-CPA-O0-NEXT:    ldrb w8, [x8]
+; CHECK-CPA-O0-NEXT:    cbz w8, .LBB14_2
+; CHECK-CPA-O0-NEXT:    b .LBB14_1
+; CHECK-CPA-O0-NEXT:  .LBB14_1:
+; CHECK-CPA-O0-NEXT:    adrp x0, .L.str
+; CHECK-CPA-O0-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-CPA-O0-NEXT:    bl printf
+; CHECK-CPA-O0-NEXT:    b .LBB14_2
+; CHECK-CPA-O0-NEXT:  .LBB14_2:
+; CHECK-CPA-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CPA-O0-NEXT:    ret
+;
+; CHECK-CPA-O3-LABEL: multidim:
+; CHECK-CPA-O3:       // %bb.0: // %entry
+; CHECK-CPA-O3-NEXT:    adrp x8, b
+; CHECK-CPA-O3-NEXT:    mov w9, #2 // =0x2
+; CHECK-CPA-O3-NEXT:    adrp x10, a
+; CHECK-CPA-O3-NEXT:    add x10, x10, :lo12:a
+; CHECK-CPA-O3-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-CPA-O3-NEXT:    addpt x9, x10, x9
+; CHECK-CPA-O3-NEXT:    addpt x9, x9, x8, lsl #1
+; CHECK-CPA-O3-NEXT:    add x8, x8, #1
+; CHECK-CPA-O3-NEXT:    addpt x8, x9, x8
+; CHECK-CPA-O3-NEXT:    ldrb w8, [x8]
+; CHECK-CPA-O3-NEXT:    cbz w8, .LBB14_2
+; CHECK-CPA-O3-NEXT:  // %bb.1:
+; CHECK-CPA-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-CPA-O3-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-CPA-O3-NEXT:    .cfi_offset w30, -16
+; CHECK-CPA-O3-NEXT:    adrp x0, .L.str
+; CHECK-CPA-O3-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-CPA-O3-NEXT:    bl printf
+; CHECK-CPA-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-CPA-O3-NEXT:  .LBB14_2:
+; CHECK-CPA-O3-NEXT:    ret
+;
+; CHECK-NOCPA-O0-LABEL: multidim:
+; CHECK-NOCPA-O0:       // %bb.0: // %entry
+; CHECK-NOCPA-O0-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOCPA-O0-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOCPA-O0-NEXT:    .cfi_offset w30, -16
+; CHECK-NOCPA-O0-NEXT:    adrp x8, b
+; CHECK-NOCPA-O0-NEXT:    ldrh w9, [x8, :lo12:b]
+; CHECK-NOCPA-O0-NEXT:    adrp x8, a
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, :lo12:a
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, w9, uxtw #1
+; CHECK-NOCPA-O0-NEXT:    add w9, w9, #1
+; CHECK-NOCPA-O0-NEXT:    mov w9, w9
+; CHECK-NOCPA-O0-NEXT:    // kill: def $x9 killed $w9
+; CHECK-NOCPA-O0-NEXT:    add x8, x8, x9
+; CHECK-NOCPA-O0-NEXT:    ldrb w8, [x8, #2]
+; CHECK-NOCPA-O0-NEXT:    cbz w8, .LBB14_2
+; CHECK-NOCPA-O0-NEXT:    b .LBB14_1
+; CHECK-NOCPA-O0-NEXT:  .LBB14_1:
+; CHECK-NOCPA-O0-NEXT:    adrp x0, .L.str
+; CHECK-NOCPA-O0-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-NOCPA-O0-NEXT:    bl printf
+; CHECK-NOCPA-O0-NEXT:    b .LBB14_2
+; CHECK-NOCPA-O0-NEXT:  .LBB14_2:
+; CHECK-NOCPA-O0-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOCPA-O0-NEXT:    ret
+;
+; CHECK-NOCPA-O3-LABEL: multidim:
+; CHECK-NOCPA-O3:       // %bb.0: // %entry
+; CHECK-NOCPA-O3-NEXT:    adrp x8, b
+; CHECK-NOCPA-O3-NEXT:    adrp x9, a
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, :lo12:a
+; CHECK-NOCPA-O3-NEXT:    ldrh w8, [x8, :lo12:b]
+; CHECK-NOCPA-O3-NEXT:    add x9, x9, x8, lsl #1
+; CHECK-NOCPA-O3-NEXT:    add x8, x9, x8
+; CHECK-NOCPA-O3-NEXT:    ldrb w8, [x8, #3]
+; CHECK-NOCPA-O3-NEXT:    cbz w8, .LBB14_2
+; CHECK-NOCPA-O3-NEXT:  // %bb.1:
+; CHECK-NOCPA-O3-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NOCPA-O3-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NOCPA-O3-NEXT:    .cfi_offset w30, -16
+; CHECK-NOCPA-O3-NEXT:    adrp x0, .L.str
+; CHECK-NOCPA-O3-NEXT:    add x0, x0, :lo12:.L.str
+; CHECK-NOCPA-O3-NEXT:    bl printf
+; CHECK-NOCPA-O3-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NOCPA-O3-NEXT:  .LBB14_2:
+; CHECK-NOCPA-O3-NEXT:    ret
+entry:
+  %1 = load i16, ptr @b, align 2
+  %2 = zext i16 %1 to i64
+  %3 = getelementptr inbounds nuw [1 x [2 x i8]], ptr getelementptr inbounds ([2 x [1 x [2 x i8]]], ptr @a, i64 0, i64 1), i64 0, i64 %2
+  %4 = load i16, ptr @b, align 2
+  %5 = zext i16 %4 to i32
+  %6 = add nsw i32 %5, 1
+  %7 = sext i32 %6 to i64
+  %8 = getelementptr inbounds [2 x i8], ptr %3, i64 0, i64 %7
+  %9 = load i8, ptr %8, align 1
+  %10 = icmp ne i8 %9, 0
+  br i1 %10, label %11, label %13
+
+11:
+  %12 = call i32 (ptr, ...) @printf(ptr noundef @.str)
+  br label %13
+
+13:
+  ret void
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr, ptr, i64, i1)
+declare i32 @printf(ptr noundef, ...) #1
+
+!6 = !{i64 0, i64 8, !7, i64 8, i64 8, !7, i64 16, i64 8, !7, i64 24, i64 8, !7, i64 32, i64 8, !7, i64 40, i64 8, !7}
+!7 = !{!8, !8, i64 0}
+!8 = !{!"long", !9, i64 0}
+!9 = !{!"omnipotent char", !10, i64 0}
+!10 = !{!"Simple C++ TBAA"}
+!11 = !{i64 0, i64 8, !7, i64 8, i64 8, !7}
-- 
cgit v1.2.3


From 491b82a5ec1add78d2c93370580a2f1897b6a364 Mon Sep 17 00:00:00 2001
From: Peter Collingbourne <peter@pcc.me.uk>
Date: Wed, 30 Apr 2025 18:25:54 -0700
Subject: ELF: Add branch-to-branch optimization.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

When code calls a function which then immediately tail calls another
function there is no need to go via the intermediate function. By
branching directly to the target function we reduce the program's working
set for a slight increase in runtime performance.

Normally it is relatively uncommon to have functions that just tail call
another function, but with LLVM control flow integrity we have jump tables
that replace the function itself as the canonical address. As a result,
when a function address is taken and called directly, for example after
a compiler optimization resolves the indirect call, or if code built
without control flow integrity calls the function, the call will go via
the jump table.

The impact of this optimization was measured using a large internal
Google benchmark. The results were as follows:

CFI enabled:  +0.1% ± 0.05% queries per second
CFI disabled: +0.01% queries per second [not statistically significant]

The optimization is enabled by default at -O2 but may also be enabled
or disabled individually with --{,no-}branch-to-branch.

This optimization is implemented for AArch64 and X86_64 only.

lld's runtime performance (real execution time) after adding this
optimization was measured using firefox-x64 from lld-speed-test [1]
with ldflags "-O2 -S" on an Apple M2 Ultra. The results are as follows:

```
    N           Min           Max        Median           Avg        Stddev
x 512     1.2264546     1.3481076     1.2970261     1.2965788   0.018620888
+ 512     1.2561196     1.3839965     1.3214632     1.3209327   0.019443971
Difference at 95.0% confidence
	0.0243538 +/- 0.00233202
	1.87831% +/- 0.179859%
	(Student's t, pooled s = 0.0190369)
```

[1] https://discourse.llvm.org/t/improving-the-reproducibility-of-linker-benchmarking/86057

Pull Request: https://github.com/llvm/llvm-project/pull/138366
---
 lld/ELF/Arch/AArch64.cpp                |  59 ++++++++++++++
 lld/ELF/Arch/TargetImpl.h               |  93 ++++++++++++++++++++++
 lld/ELF/Arch/X86_64.cpp                 |  69 +++++++++++++++++
 lld/ELF/Config.h                        |   1 +
 lld/ELF/Driver.cpp                      |   2 +
 lld/ELF/InputSection.cpp                |   5 +-
 lld/ELF/Options.td                      |   4 +
 lld/ELF/Relocations.cpp                 |   8 +-
 lld/ELF/Target.h                        |   1 +
 lld/docs/ReleaseNotes.rst               |   4 +
 lld/docs/ld.lld.1                       |   9 ++-
 lld/test/ELF/aarch64-branch-to-branch.s |  82 ++++++++++++++++++++
 lld/test/ELF/x86-64-branch-to-branch.s  | 133 ++++++++++++++++++++++++++++++++
 13 files changed, 464 insertions(+), 6 deletions(-)
 create mode 100644 lld/ELF/Arch/TargetImpl.h
 create mode 100644 lld/test/ELF/aarch64-branch-to-branch.s
 create mode 100644 lld/test/ELF/x86-64-branch-to-branch.s

diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp
index 8a225ed103ee..f00c91b5886f 100644
--- a/lld/ELF/Arch/AArch64.cpp
+++ b/lld/ELF/Arch/AArch64.cpp
@@ -11,6 +11,7 @@
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "TargetImpl.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Endian.h"
 
@@ -82,6 +83,7 @@ public:
                 uint64_t val) const override;
   RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override;
   void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override;
+  void applyBranchToBranchOpt() const override;
 
 private:
   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
@@ -974,6 +976,63 @@ void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   }
 }
 
+static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
+                                                        Relocation &r) {
+  // Identify a control transfer relocation for the branch-to-branch
+  // optimization. A "control transfer relocation" means a B or BL
+  // target but it also includes relative vtable relocations for example.
+  //
+  // We require the relocation type to be JUMP26, CALL26 or PLT32. With a
+  // relocation type of PLT32 the value may be assumed to be used for branching
+  // directly to the symbol and the addend is only used to produce the relocated
+  // value (hence the effective addend is always 0). This is because if a PLT is
+  // needed the addend will be added to the address of the PLT, and it doesn't
+  // make sense to branch into the middle of a PLT. For example, relative vtable
+  // relocations use PLT32 and 0 or a positive value as the addend but still are
+  // used to branch to the symbol.
+  //
+  // With JUMP26 or CALL26 the only reasonable interpretation of a non-zero
+  // addend is that we are branching to symbol+addend so that becomes the
+  // effective addend.
+  if (r.type == R_AARCH64_PLT32)
+    return 0;
+  if (r.type == R_AARCH64_JUMP26 || r.type == R_AARCH64_CALL26)
+    return r.addend;
+  return std::nullopt;
+}
+
+static std::pair<Relocation *, uint64_t>
+getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
+  auto *i =
+      std::partition_point(is.relocations.begin(), is.relocations.end(),
+                           [&](Relocation &r) { return r.offset < offset; });
+  if (i != is.relocations.end() && i->offset == offset &&
+      i->type == R_AARCH64_JUMP26) {
+    return {i, i->addend};
+  }
+  return {nullptr, 0};
+}
+
+static void redirectControlTransferRelocations(Relocation &r1,
+                                               const Relocation &r2) {
+  r1.expr = r2.expr;
+  r1.sym = r2.sym;
+  // With PLT32 we must respect the original addend as that affects the value's
+  // interpretation. With the other relocation types the original addend is
+  // irrelevant because it referred to an offset within the original target
+  // section so we overwrite it.
+  if (r1.type == R_AARCH64_PLT32)
+    r1.addend += r2.addend;
+  else
+    r1.addend = r2.addend;
+}
+
+void AArch64::applyBranchToBranchOpt() const {
+  applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
+                             getBranchInfoAtTarget,
+                             redirectControlTransferRelocations);
+}
+
 // AArch64 may use security features in variant PLT sequences. These are:
 // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target
 // Indicator (BTI) introduced in armv8.5-a. The additional instructions used
diff --git a/lld/ELF/Arch/TargetImpl.h b/lld/ELF/Arch/TargetImpl.h
new file mode 100644
index 000000000000..f1206570d3e3
--- /dev/null
+++ b/lld/ELF/Arch/TargetImpl.h
@@ -0,0 +1,93 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLD_ELF_ARCH_TARGETIMPL_H
+#define LLD_ELF_ARCH_TARGETIMPL_H
+
+#include "InputFiles.h"
+#include "InputSection.h"
+#include "Relocations.h"
+#include "Symbols.h"
+#include "llvm/BinaryFormat/ELF.h"
+
+namespace lld::elf {
+
+// getControlTransferAddend: If this relocation is used for control transfer
+// instructions (e.g. branch, branch-link or call) or code references (e.g.
+// virtual function pointers) and indicates an address-insignificant reference,
+// return the effective addend for the relocation, otherwise return
+// std::nullopt. The effective addend for a relocation is the addend that is
+// used to determine its branch destination.
+//
+// getBranchInfoAtTarget: If a control transfer relocation referring to
+// is+offset directly transfers control to a relocated branch instruction in the
+// specified section, return the relocation for the branch target as well as its
+// effective addend (see above). Otherwise return {nullptr, 0}.
+//
+// redirectControlTransferRelocations: Given r1, a relocation for which
+// getControlTransferAddend() returned a value, and r2, a relocation returned by
+// getBranchInfo(), modify r1 so that it branches directly to the target of r2.
+template <typename GetControlTransferAddend, typename GetBranchInfoAtTarget,
+          typename RedirectControlTransferRelocations>
+inline void applyBranchToBranchOptImpl(
+    Ctx &ctx, GetControlTransferAddend getControlTransferAddend,
+    GetBranchInfoAtTarget getBranchInfoAtTarget,
+    RedirectControlTransferRelocations redirectControlTransferRelocations) {
+  // Needs to run serially because it writes to the relocations array as well as
+  // reading relocations of other sections.
+  for (ELFFileBase *f : ctx.objectFiles) {
+    auto getRelocBranchInfo =
+        [&getBranchInfoAtTarget](
+            Relocation &r,
+            uint64_t addend) -> std::pair<Relocation *, uint64_t> {
+      auto *target = dyn_cast_or_null<Defined>(r.sym);
+      // We don't allow preemptible symbols or ifuncs (may go somewhere else),
+      // absolute symbols (runtime behavior unknown), non-executable or writable
+      // memory (ditto) or non-regular sections (no section data).
+      if (!target || target->isPreemptible || target->isGnuIFunc() ||
+          !target->section ||
+          !(target->section->flags & llvm::ELF::SHF_EXECINSTR) ||
+          (target->section->flags & llvm::ELF::SHF_WRITE) ||
+          target->section->kind() != SectionBase::Regular)
+        return {nullptr, 0};
+      return getBranchInfoAtTarget(*cast<InputSection>(target->section),
+                                   target->value + addend);
+    };
+    for (InputSectionBase *s : f->getSections()) {
+      if (!s)
+        continue;
+      for (Relocation &r : s->relocations) {
+        std::optional<uint64_t> addend =
+            getControlTransferAddend(*cast<InputSection>(s), r);
+        if (!addend)
+          continue;
+        std::pair<Relocation *, uint64_t> targetAndAddend =
+            getRelocBranchInfo(r, *addend);
+        if (!targetAndAddend.first)
+          continue;
+        // Avoid getting stuck in an infinite loop if we encounter a branch
+        // that (possibly indirectly) branches to itself. It is unlikely
+        // that more than 5 iterations will ever be needed in practice.
+        size_t iterations = 5;
+        while (iterations--) {
+          std::pair<Relocation *, uint64_t> nextTargetAndAddend =
+              getRelocBranchInfo(*targetAndAddend.first,
+                                 targetAndAddend.second);
+          if (!nextTargetAndAddend.first)
+            break;
+          targetAndAddend = nextTargetAndAddend;
+        }
+        redirectControlTransferRelocations(r, *targetAndAddend.first);
+      }
+    }
+  }
+}
+
+} // namespace lld::elf
+
+#endif
diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp
index 974da4d96320..b991b6f905b9 100644
--- a/lld/ELF/Arch/X86_64.cpp
+++ b/lld/ELF/Arch/X86_64.cpp
@@ -11,6 +11,7 @@
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
+#include "TargetImpl.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MathExtras.h"
@@ -49,6 +50,7 @@ public:
   bool deleteFallThruJmpInsn(InputSection &is, InputFile *file,
                              InputSection *nextIS) const override;
   bool relaxOnce(int pass) const override;
+  void applyBranchToBranchOpt() const override;
 
 private:
   void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const;
@@ -1161,6 +1163,73 @@ void X86_64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
   }
 }
 
+static std::optional<uint64_t> getControlTransferAddend(InputSection &is,
+                                                        Relocation &r) {
+  // Identify a control transfer relocation for the branch-to-branch
+  // optimization. A "control transfer relocation" usually means a CALL or JMP
+  // target but it also includes relative vtable relocations for example.
+  //
+  // We require the relocation type to be PLT32. With a relocation type of PLT32
+  // the value may be assumed to be used for branching directly to the symbol
+  // and the addend is only used to produce the relocated value (hence the
+  // effective addend is always 0). This is because if a PLT is needed the
+  // addend will be added to the address of the PLT, and it doesn't make sense
+  // to branch into the middle of a PLT. For example, relative vtable
+  // relocations use PLT32 and 0 or a positive value as the addend but still are
+  // used to branch to the symbol.
+  //
+  // STT_SECTION symbols are a special case on x86 because the LLVM assembler
+  // uses them for branches to local symbols which are assembled as referring to
+  // the section symbol with the addend equal to the symbol value - 4.
+  if (r.type == R_X86_64_PLT32) {
+    if (r.sym->isSection())
+      return r.addend + 4;
+    return 0;
+  }
+  return std::nullopt;
+}
+
+static std::pair<Relocation *, uint64_t>
+getBranchInfoAtTarget(InputSection &is, uint64_t offset) {
+  auto content = is.contentMaybeDecompress();
+  if (content.size() > offset && content[offset] == 0xe9) { // JMP immediate
+    auto *i = std::partition_point(
+        is.relocations.begin(), is.relocations.end(),
+        [&](Relocation &r) { return r.offset < offset + 1; });
+    // Unlike with getControlTransferAddend() it is valid to accept a PC32
+    // relocation here because we know that this is actually a JMP and not some
+    // other reference, so the interpretation is that we add 4 to the addend and
+    // use that as the effective addend.
+    if (i != is.relocations.end() && i->offset == offset + 1 &&
+        (i->type == R_X86_64_PC32 || i->type == R_X86_64_PLT32)) {
+      return {i, i->addend + 4};
+    }
+  }
+  return {nullptr, 0};
+}
+
+static void redirectControlTransferRelocations(Relocation &r1,
+                                               const Relocation &r2) {
+  // The isSection() check handles the STT_SECTION case described above.
+  // In that case the original addend is irrelevant because it referred to an
+  // offset within the original target section so we overwrite it.
+  //
+  // The +4 is here to compensate for r2.addend which will likely be -4,
+  // but may also be addend-4 in case of a PC32 branch to symbol+addend.
+  if (r1.sym->isSection())
+    r1.addend = r2.addend;
+  else
+    r1.addend += r2.addend + 4;
+  r1.expr = r2.expr;
+  r1.sym = r2.sym;
+}
+
+void X86_64::applyBranchToBranchOpt() const {
+  applyBranchToBranchOptImpl(ctx, getControlTransferAddend,
+                             getBranchInfoAtTarget,
+                             redirectControlTransferRelocations);
+}
+
 // If Intel Indirect Branch Tracking is enabled, we have to emit special PLT
 // entries containing endbr64 instructions. A PLT entry will be split into two
 // parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt).
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 2b72d54ba410..88bda41d3648 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -302,6 +302,7 @@ struct Config {
   bool bpFunctionOrderForCompression = false;
   bool bpDataOrderForCompression = false;
   bool bpVerboseSectionOrderer = false;
+  bool branchToBranch = false;
   bool checkSections;
   bool checkDynamicRelocs;
   std::optional<llvm::DebugCompressionType> compressDebugSections;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 7e132a387a04..1e0b5988343a 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -1644,6 +1644,8 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) {
   ctx.arg.zWxneeded = hasZOption(args, "wxneeded");
   setUnresolvedSymbolPolicy(ctx, args);
   ctx.arg.power10Stubs = args.getLastArgValue(OPT_power10_stubs_eq) != "no";
+  ctx.arg.branchToBranch = args.hasFlag(
+      OPT_branch_to_branch, OPT_no_branch_to_branch, ctx.arg.optimize >= 2);
 
   if (opt::Arg *arg = args.getLastArg(OPT_eb, OPT_el)) {
     if (arg->getOption().matches(OPT_eb))
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index 0ce0f08d0387..f8786265029e 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -430,8 +430,9 @@ InputSectionBase *InputSection::getRelocatedSection() const {
 
 template <class ELFT, class RelTy>
 void InputSection::copyRelocations(Ctx &ctx, uint8_t *buf) {
-  if (ctx.arg.relax && !ctx.arg.relocatable &&
-      (ctx.arg.emachine == EM_RISCV || ctx.arg.emachine == EM_LOONGARCH)) {
+  bool linkerRelax =
+      ctx.arg.relax && is_contained({EM_RISCV, EM_LOONGARCH}, ctx.arg.emachine);
+  if (!ctx.arg.relocatable && (linkerRelax || ctx.arg.branchToBranch)) {
     // On LoongArch and RISC-V, relaxation might change relocations: copy
     // from internal ones that are updated by relaxation.
     InputSectionBase *sec = getRelocatedSection();
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index c795147eb966..d7e331316700 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -59,6 +59,10 @@ def build_id: J<"build-id=">, HelpText<"Generate build ID note">,
   MetaVarName<"[fast,md5,sha1,uuid,0x<hexstring>]">;
 def : F<"build-id">, Alias<build_id>, AliasArgs<["sha1"]>, HelpText<"Alias for --build-id=sha1">;
 
+defm branch_to_branch: BB<"branch-to-branch",
+    "Enable branch-to-branch optimization (default at -O2)",
+    "Disable branch-to-branch optimization (default at -O0 and -O1)">;
+
 defm check_sections: B<"check-sections",
     "Check section addresses for overlaps (default)",
     "Do not check section addresses for overlaps">;
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 6c4209a2b81e..43f19186f098 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -1665,9 +1665,10 @@ void RelocationScanner::scan(Relocs<RelTy> rels) {
   }
 
   // Sort relocations by offset for more efficient searching for
-  // R_RISCV_PCREL_HI20 and R_PPC64_ADDR64.
+  // R_RISCV_PCREL_HI20, R_PPC64_ADDR64 and the branch-to-branch optimization.
   if (ctx.arg.emachine == EM_RISCV ||
-      (ctx.arg.emachine == EM_PPC64 && sec->name == ".toc"))
+      (ctx.arg.emachine == EM_PPC64 && sec->name == ".toc") ||
+      ctx.arg.branchToBranch)
     llvm::stable_sort(sec->relocs(),
                       [](const Relocation &lhs, const Relocation &rhs) {
                         return lhs.offset < rhs.offset;
@@ -1958,6 +1959,9 @@ void elf::postScanRelocations(Ctx &ctx) {
   for (ELFFileBase *file : ctx.objectFiles)
     for (Symbol *sym : file->getLocalSymbols())
       fn(*sym);
+
+  if (ctx.arg.branchToBranch)
+    ctx.target->applyBranchToBranchOpt();
 }
 
 static bool mergeCmp(const InputSection *a, const InputSection *b) {
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index fd1e5d33c438..6dd20b2f0cba 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -101,6 +101,7 @@ public:
 
   virtual void applyJumpInstrMod(uint8_t *loc, JumpModType type,
                                  JumpModType val) const {}
+  virtual void applyBranchToBranchOpt() const {}
 
   virtual ~TargetInfo();
 
diff --git a/lld/docs/ReleaseNotes.rst b/lld/docs/ReleaseNotes.rst
index 064ed0828c31..dabfc961dd5b 100644
--- a/lld/docs/ReleaseNotes.rst
+++ b/lld/docs/ReleaseNotes.rst
@@ -62,6 +62,10 @@ ELF Improvements
   on executable sections.
   (`#128883 <https://github.com/llvm/llvm-project/pull/128883>`_)
 
+* For AArch64 and X86_64, added ``--branch-to-branch``, which rewrites branches
+  that point to another branch instruction to instead branch directly to the
+  target of the second instruction. Enabled by default at ``-O2``.
+  
 Breaking changes
 ----------------
 * Executable-only and readable-executable sections are now allowed to be placed
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index cfacdb081a80..7edc522b4f6a 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -93,6 +93,11 @@ Bind default visibility defined STB_GLOBAL function symbols locally for
 .Fl shared.
 .It Fl -be8
 Write a Big Endian ELF File using BE8 format(AArch32 only)
+.It Fl -branch-to-branch
+Enable the branch-to-branch optimizations: a branch whose target is
+another branch instruction is rewritten to point to the latter branch
+target (AArch64 and X86_64 only). Enabled by default at
+.Fl O2 Ns .
 .It Fl -build-id Ns = Ns Ar value
 Generate a build ID note.
 .Ar value
@@ -414,7 +419,7 @@ If not specified,
 .Dv a.out
 is used as a default.
 .It Fl O Ns Ar value
-Optimize output file size.
+Optimize output file.
 .Ar value
 may be:
 .Pp
@@ -424,7 +429,7 @@ Disable string merging.
 .It Cm 1
 Enable string merging.
 .It Cm 2
-Enable string tail merging.
+Enable string tail merging and branch-to-branch optimization.
 .El
 .Pp
 .Fl O Ns Cm 1
diff --git a/lld/test/ELF/aarch64-branch-to-branch.s b/lld/test/ELF/aarch64-branch-to-branch.s
new file mode 100644
index 000000000000..7dc485aef853
--- /dev/null
+++ b/lld/test/ELF/aarch64-branch-to-branch.s
@@ -0,0 +1,82 @@
+# REQUIRES: aarch64
+
+## Test that the branch-to-branch optimization follows the links
+## from f1 -> f2 -> f3 and updates all references to point to f3.
+
+# RUN: llvm-mc -filetype=obj -triple=aarch64-pc-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t --branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+
+## Test that branch-to-branch is disabled by default.
+
+# RUN: ld.lld %t.o -o %t --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --no-branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+## Test that branch-to-branch is disabled for preemptible symbols.
+
+# RUN: ld.lld %t.o -o %t --branch-to-branch -shared --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+.section .rodata.vtable,"a"
+.globl vtable
+vtable:
+# B2B: Contents of section .rodata:
+# RELOC: RELOCATION RECORDS FOR [.rodata]:
+# RELOC-NEXT: OFFSET
+# B2B-NEXT: [[VF:[0-9a-f]{8}]]
+# B2B-RELOC-NEXT: R_AARCH64_PLT32 f3
+# NOB2B-RELOC-NEXT: R_AARCH64_PLT32 f1
+.4byte f1@PLT - vtable
+# B2B-SAME: [[VF]]
+# B2B-RELOC-NEXT: R_AARCH64_PLT32 f3+0x4
+# NOB2B-RELOC-NEXT: R_AARCH64_PLT32 f2+0x4
+.4byte f2@PLT - vtable
+# B2B-SAME: [[VF]]
+# RELOC-NEXT: R_AARCH64_PLT32 f3+0x8
+.4byte f3@PLT - vtable
+
+.section .text._start,"ax"
+.globl _start
+# CHECK: <_start>:
+# RELOC: RELOCATION RECORDS FOR [.text]:
+# RELOC-NEXT: OFFSET
+_start:
+# B2B: bl {{.*}} <f3>
+# B2B-RELOC-NEXT: R_AARCH64_CALL26 f3
+# NOB2B: bl {{.*}} <f1{{.*}}>
+# NOB2B-RELOC-NEXT: R_AARCH64_CALL26 f1
+bl f1
+# B2B: b {{.*}} <f3>
+# B2B-RELOC-NEXT: R_AARCH64_JUMP26 f3
+# NOB2B: b {{.*}} <f2{{.*}}>
+# NOB2B-RELOC-NEXT: R_AARCH64_JUMP26 f2
+b f2
+
+.section .text.f1,"ax"
+.globl f1
+f1:
+# B2B-RELOC-NEXT: R_AARCH64_JUMP26 f3
+# NOB2B-RELOC-NEXT: R_AARCH64_JUMP26 f2
+b f2
+
+.section .text.f2,"ax"
+.globl f2
+# CHECK: <f2>:
+f2:
+# CHECK-NEXT: b {{.*}} <f3{{.*}}>
+# RELOC-NEXT: R_AARCH64_JUMP26 f3
+b f3
+
+.section .text.f3,"ax"
+.globl f3
+f3:
+ret
diff --git a/lld/test/ELF/x86-64-branch-to-branch.s b/lld/test/ELF/x86-64-branch-to-branch.s
new file mode 100644
index 000000000000..dabf5be571ec
--- /dev/null
+++ b/lld/test/ELF/x86-64-branch-to-branch.s
@@ -0,0 +1,133 @@
+# REQUIRES: x86
+
+## Test that the branch-to-branch optimization follows the links
+## from f1 -> f2 -> f3 and updates all references to point to f3.
+ 
+# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
+# RUN: ld.lld %t.o -o %t --branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,B2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,B2B-RELOC %s
+
+## Test that branch-to-branch is disabled by default.
+
+# RUN: ld.lld %t.o -o %t --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+# RUN: ld.lld %t.o -o %t -O2 --no-branch-to-branch --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+## Test that branch-to-branch is disabled for preemptible symbols.
+
+# RUN: ld.lld %t.o -o %t --branch-to-branch -shared --emit-relocs
+# RUN: llvm-objdump -d -s %t | FileCheck --check-prefixes=CHECK,NOB2B %s
+# RUN: llvm-objdump -r %t | FileCheck --check-prefixes=RELOC,NOB2B-RELOC %s
+
+.section .rodata.vtable,"a"
+.globl vtable
+vtable:
+# B2B: Contents of section .rodata:
+# RELOC: RELOCATION RECORDS FOR [.rodata]:
+# RELOC-NEXT: OFFSET
+# B2B-NEXT: [[VF:[0-9a-f]{8}]]
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f1
+.4byte f1@PLT - vtable
+# B2B-SAME: [[VF]]
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3+0x4
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2+0x4
+.4byte f2@PLT - vtable
+# B2B-SAME: [[VF]]
+# RELOC-NEXT: R_X86_64_PLT32 f3+0x8
+.4byte f3@PLT - vtable
+
+# For .rodata.f6
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+ 
+.section .text._start,"ax"
+.globl _start
+# CHECK: <_start>:
+# RELOC: RELOCATION RECORDS FOR [.text]:
+# RELOC-NEXT: OFFSET
+_start:
+# B2B-NEXT: jmp {{.*}} <f3>
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-NEXT: jmp {{.*}} <f1{{.*}}>
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f1-0x4
+jmp f1
+# B2B-NEXT: jmp {{.*}} <f3>
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-NEXT: jmp {{.*}} <f2{{.*}}>
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2-0x4
+jmp f2
+# This will assemble to a relocation pointing to an STT_SECTION for .text.f4
+# with an addend, which looks similar to the relative vtable cases above but
+# requires different handling of the addend so that we don't think this is
+# branching to the `jmp f3` at the start of the target section.
+# CHECK-NEXT: jmp {{.*}} <f4{{.*}}>
+# RELOC-NEXT: R_X86_64_PLT32 .text+0x2e
+jmp f4
+# B2B-NEXT: jmp 0x[[IPLT:[0-9a-f]*]]
+# RELOC-NEXT: R_X86_64_PLT32 f5-0x4
+jmp f5
+# B2B-NEXT: jmp {{.*}} <f6>
+# RELOC-NEXT: R_X86_64_PLT32 f6-0x4
+jmp f6
+# B2B-NEXT: jmp {{.*}} <f7>
+# RELOC-NEXT: R_X86_64_PLT32 f7-0x4
+jmp f7
+
+.section .text.f1,"ax"
+.globl f1
+f1:
+# B2B-RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+# NOB2B-RELOC-NEXT: R_X86_64_PLT32 f2-0x4
+jmp f2
+
+.section .text.f2,"ax"
+.globl f2
+# CHECK: <f2>:
+f2:
+# CHECK-NEXT: jmp {{.*}} <f3{{.*}}>
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .text.f3,"ax"
+.globl f3
+f3:
+# Test that a self-branch doesn't trigger an infinite loop.
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .text.f4,"ax"
+jmp f3
+f4:
+ret
+
+.section .text.f5,"ax"
+.type f5, @gnu_indirect_function
+.globl f5
+f5:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+.section .rodata.f6,"a"
+.globl f6
+f6:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+# RELOC: RELOCATION RECORDS FOR [.wtext.f7]:
+# RELOC-NEXT: OFFSET
+
+.section .wtext.f7,"awx"
+.globl f7
+f7:
+# RELOC-NEXT: R_X86_64_PLT32 f3-0x4
+jmp f3
+
+# B2B: <.iplt>:
+# B2B-NEXT: [[IPLT]]:
-- 
cgit v1.2.3


From 6110dead894bec37d6373eda4cba2d5dc426f824 Mon Sep 17 00:00:00 2001
From: Michael Spencer <bigcheesegs@gmail.com>
Date: Fri, 20 Jun 2025 13:28:05 -0700
Subject: [clang][scan-deps] Add option to disable caching stat failures
 (#144000)

While the source code isn't supposed to change during a build, in some
environments it does. This adds an option that disables caching of stat
failures, meaning that source files can be added to the build during
scanning.

This adds a `-no-cache-negative-stats` option to clang-scan-deps to
enable this behavior. There are no tests for clang-scan-deps as there's
no reliable way to do so from it. A unit test has been added that
modifies the filesystem between scans to test it.
---
 .../DependencyScanningFilesystem.h                 | 24 ++++------
 .../DependencyScanning/DependencyScanningService.h |  6 ++-
 .../DependencyScanningFilesystem.cpp               | 40 ++++++++++++++---
 .../DependencyScanningService.cpp                  |  3 +-
 .../DependencyScanningWorker.cpp                   |  3 +-
 clang/tools/clang-scan-deps/ClangScanDeps.cpp      |  8 +++-
 clang/tools/clang-scan-deps/Opts.td                |  1 +
 .../DependencyScanning/DependencyScannerTest.cpp   | 52 ++++++++++++++++++++++
 .../DependencyScanningFilesystemTest.cpp           | 37 ++++++++-------
 9 files changed, 132 insertions(+), 42 deletions(-)

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index a20a89a4c2b7..da83220babea 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -23,6 +23,8 @@ namespace clang {
 namespace tooling {
 namespace dependencies {
 
+class DependencyScanningService;
+
 using DependencyDirectivesTy =
     SmallVector<dependency_directives_scan::Directive, 20>;
 
@@ -349,7 +351,7 @@ public:
   static const char ID;
 
   DependencyScanningWorkerFilesystem(
-      DependencyScanningFilesystemSharedCache &SharedCache,
+      DependencyScanningService &Service,
       IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);
 
   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
@@ -435,10 +437,7 @@ private:
   /// Returns entry associated with the unique ID in the shared cache or nullptr
   /// if none is found.
   const CachedFileSystemEntry *
-  findSharedEntryByUID(llvm::vfs::Status Stat) const {
-    return SharedCache.getShardForUID(Stat.getUniqueID())
-        .findEntryByUID(Stat.getUniqueID());
-  }
+  findSharedEntryByUID(llvm::vfs::Status Stat) const;
 
   /// Associates the given entry with the filename in the local cache and
   /// returns it.
@@ -452,20 +451,14 @@ private:
   /// some. Otherwise, constructs new one with the given error code, associates
   /// it with the filename and returns the result.
   const CachedFileSystemEntry &
-  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC) {
-    return SharedCache.getShardForFilename(Filename)
-        .getOrEmplaceEntryForFilename(Filename, EC);
-  }
+  getOrEmplaceSharedEntryForFilename(StringRef Filename, std::error_code EC);
 
   /// Returns entry associated with the filename in the shared cache if there is
   /// some. Otherwise, associates the given entry with the filename and returns
   /// it.
   const CachedFileSystemEntry &
   getOrInsertSharedEntryForFilename(StringRef Filename,
-                                    const CachedFileSystemEntry &Entry) {
-    return SharedCache.getShardForFilename(Filename)
-        .getOrInsertEntryForFilename(Filename, Entry);
-  }
+                                    const CachedFileSystemEntry &Entry);
 
   void printImpl(raw_ostream &OS, PrintType Type,
                  unsigned IndentLevel) const override {
@@ -478,8 +471,9 @@ private:
   /// VFS.
   bool shouldBypass(StringRef Path) const;
 
-  /// The global cache shared between worker threads.
-  DependencyScanningFilesystemSharedCache &SharedCache;
+  /// The service associated with this VFS.
+  DependencyScanningService &Service;
+
   /// The local cache is used by the worker thread to cache file system queries
   /// locally instead of querying the global cache every time.
   DependencyScanningFilesystemLocalCache LocalCache;
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index 4e97c7bc9f36..ceaf3c2279e7 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -87,7 +87,8 @@ public:
       ScanningOptimizations OptimizeArgs = ScanningOptimizations::Default,
       bool EagerLoadModules = false, bool TraceVFS = false,
       std::time_t BuildSessionTimestamp =
-          llvm::sys::toTimeT(std::chrono::system_clock::now()));
+          llvm::sys::toTimeT(std::chrono::system_clock::now()),
+      bool CacheNegativeStats = true);
 
   ScanningMode getMode() const { return Mode; }
 
@@ -99,6 +100,8 @@ public:
 
   bool shouldTraceVFS() const { return TraceVFS; }
 
+  bool shouldCacheNegativeStats() const { return CacheNegativeStats; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
@@ -116,6 +119,7 @@ private:
   const bool EagerLoadModules;
   /// Whether to trace VFS accesses.
   const bool TraceVFS;
+  const bool CacheNegativeStats;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
   /// The global module cache entries.
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 140833050f4e..2868522f8001 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Threading.h"
 #include <optional>
@@ -232,19 +233,19 @@ bool DependencyScanningWorkerFilesystem::shouldBypass(StringRef Path) const {
 }
 
 DependencyScanningWorkerFilesystem::DependencyScanningWorkerFilesystem(
-    DependencyScanningFilesystemSharedCache &SharedCache,
+    DependencyScanningService &Service,
     IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
     : llvm::RTTIExtends<DependencyScanningWorkerFilesystem,
                         llvm::vfs::ProxyFileSystem>(std::move(FS)),
-      SharedCache(SharedCache),
-      WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
+      Service(Service), WorkingDirForCacheLookup(llvm::errc::invalid_argument) {
   updateWorkingDirForCacheLookup();
 }
 
 const CachedFileSystemEntry &
 DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForUID(
     TentativeEntry TEntry) {
-  auto &Shard = SharedCache.getShardForUID(TEntry.Status.getUniqueID());
+  auto &Shard =
+      Service.getSharedCache().getShardForUID(TEntry.Status.getUniqueID());
   return Shard.getOrEmplaceEntryForUID(TEntry.Status.getUniqueID(),
                                        std::move(TEntry.Status),
                                        std::move(TEntry.Contents));
@@ -255,18 +256,44 @@ DependencyScanningWorkerFilesystem::findEntryByFilenameWithWriteThrough(
     StringRef Filename) {
   if (const auto *Entry = LocalCache.findEntryByFilename(Filename))
     return Entry;
-  auto &Shard = SharedCache.getShardForFilename(Filename);
+  auto &Shard = Service.getSharedCache().getShardForFilename(Filename);
   if (const auto *Entry = Shard.findEntryByFilename(Filename))
     return &LocalCache.insertEntryForFilename(Filename, *Entry);
   return nullptr;
 }
 
+const CachedFileSystemEntry *
+DependencyScanningWorkerFilesystem::findSharedEntryByUID(
+    llvm::vfs::Status Stat) const {
+  return Service.getSharedCache()
+      .getShardForUID(Stat.getUniqueID())
+      .findEntryByUID(Stat.getUniqueID());
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrEmplaceSharedEntryForFilename(
+    StringRef Filename, std::error_code EC) {
+  return Service.getSharedCache()
+      .getShardForFilename(Filename)
+      .getOrEmplaceEntryForFilename(Filename, EC);
+}
+
+const CachedFileSystemEntry &
+DependencyScanningWorkerFilesystem::getOrInsertSharedEntryForFilename(
+    StringRef Filename, const CachedFileSystemEntry &Entry) {
+  return Service.getSharedCache()
+      .getShardForFilename(Filename)
+      .getOrInsertEntryForFilename(Filename, Entry);
+}
+
 llvm::ErrorOr<const CachedFileSystemEntry &>
 DependencyScanningWorkerFilesystem::computeAndStoreResult(
     StringRef OriginalFilename, StringRef FilenameForLookup) {
   llvm::ErrorOr<llvm::vfs::Status> Stat =
       getUnderlyingFS().status(OriginalFilename);
   if (!Stat) {
+    if (!Service.shouldCacheNegativeStats())
+      return Stat.getError();
     const auto &Entry =
         getOrEmplaceSharedEntryForFilename(FilenameForLookup, Stat.getError());
     return insertLocalEntryForFilename(FilenameForLookup, Entry);
@@ -420,7 +447,8 @@ DependencyScanningWorkerFilesystem::getRealPath(const Twine &Path,
     return HandleCachedRealPath(*RealPath);
 
   // If we have the result in the shared cache, cache it locally.
-  auto &Shard = SharedCache.getShardForFilename(*FilenameForLookup);
+  auto &Shard =
+      Service.getSharedCache().getShardForFilename(*FilenameForLookup);
   if (const auto *ShardRealPath =
           Shard.findRealPathByFilename(*FilenameForLookup)) {
     const auto &RealPath = LocalCache.insertRealPathForFilename(
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index 7f40c99f0728..c2f3cdbb02e3 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -15,7 +15,8 @@ using namespace dependencies;
 DependencyScanningService::DependencyScanningService(
     ScanningMode Mode, ScanningOutputFormat Format,
     ScanningOptimizations OptimizeArgs, bool EagerLoadModules, bool TraceVFS,
-    std::time_t BuildSessionTimestamp)
+    std::time_t BuildSessionTimestamp, bool CacheNegativeStats)
     : Mode(Mode), Format(Format), OptimizeArgs(OptimizeArgs),
       EagerLoadModules(EagerLoadModules), TraceVFS(TraceVFS),
+      CacheNegativeStats(CacheNegativeStats),
       BuildSessionTimestamp(BuildSessionTimestamp) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index 9bd85479d981..d9820ca3c584 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -611,8 +611,7 @@ DependencyScanningWorker::DependencyScanningWorker(
 
   switch (Service.getMode()) {
   case ScanningMode::DependencyDirectivesScan:
-    DepFS =
-        new DependencyScanningWorkerFilesystem(Service.getSharedCache(), FS);
+    DepFS = new DependencyScanningWorkerFilesystem(Service, FS);
     BaseFS = DepFS;
     break;
   case ScanningMode::CanonicalPreprocessing:
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index 921ba7aadd67..ce0770a51d65 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -85,6 +85,7 @@ static ScanningOutputFormat Format = ScanningOutputFormat::Make;
 static ScanningOptimizations OptimizeArgs;
 static std::string ModuleFilesDir;
 static bool EagerLoadModules;
+static bool CacheNegativeStats = true;
 static unsigned NumThreads = 0;
 static std::string CompilationDB;
 static std::optional<std::string> ModuleName;
@@ -191,6 +192,8 @@ static void ParseArgs(int argc, char **argv) {
 
   EagerLoadModules = Args.hasArg(OPT_eager_load_pcm);
 
+  CacheNegativeStats = !Args.hasArg(OPT_no_cache_negative_stats);
+
   if (const llvm::opt::Arg *A = Args.getLastArg(OPT_j)) {
     StringRef S{A->getValue()};
     if (!llvm::to_integer(S, NumThreads, 0)) {
@@ -1080,8 +1083,9 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
     });
   };
 
-  DependencyScanningService Service(ScanMode, Format, OptimizeArgs,
-                                    EagerLoadModules, /*TraceVFS=*/Verbose);
+  DependencyScanningService Service(
+      ScanMode, Format, OptimizeArgs, EagerLoadModules, /*TraceVFS=*/Verbose,
+      llvm::sys::toTimeT(std::chrono::system_clock::now()), CacheNegativeStats);
 
   llvm::Timer T;
   T.startTimer();
diff --git a/clang/tools/clang-scan-deps/Opts.td b/clang/tools/clang-scan-deps/Opts.td
index 9cccbb3aaf0c..582ae60851e1 100644
--- a/clang/tools/clang-scan-deps/Opts.td
+++ b/clang/tools/clang-scan-deps/Opts.td
@@ -22,6 +22,7 @@ defm module_files_dir : Eq<"module-files-dir",
 
 def optimize_args_EQ : CommaJoined<["-", "--"], "optimize-args=">, HelpText<"Which command-line arguments of modules to optimize">;
 def eager_load_pcm : F<"eager-load-pcm", "Load PCM files eagerly (instead of lazily on import)">;
+def no_cache_negative_stats : F<"no-cache-negative-stats", "Don't cache stat failures">;
 
 def j : Arg<"j", "Number of worker threads to use (default: use all concurrent threads)">;
 
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
index 683d9070b1dc..d194b2877ad8 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScannerTest.cpp
@@ -384,3 +384,55 @@ TEST(DependencyScanner, ScanDepsWithDiagConsumer) {
     EXPECT_TRUE(DiagConsumer.Finished);
   }
 }
+
+TEST(DependencyScanner, NoNegativeCache) {
+  StringRef CWD = "/root";
+
+  auto VFS = new llvm::vfs::InMemoryFileSystem();
+  VFS->setCurrentWorkingDirectory(CWD);
+  auto Sept = llvm::sys::path::get_separator();
+  std::string HeaderPath =
+      std::string(llvm::formatv("{0}root{0}header.h", Sept));
+  std::string Test0Path =
+      std::string(llvm::formatv("{0}root{0}test0.cpp", Sept));
+  std::string Test1Path =
+      std::string(llvm::formatv("{0}root{0}test1.cpp", Sept));
+
+  VFS->addFile(Test0Path, 0,
+               llvm::MemoryBuffer::getMemBuffer(
+                   "#if __has_include(\"header.h\")\n#endif"));
+  VFS->addFile(Test1Path, 0,
+               llvm::MemoryBuffer::getMemBuffer("#include \"header.h\""));
+
+  DependencyScanningService Service(
+      ScanningMode::DependencyDirectivesScan, ScanningOutputFormat::Make,
+      ScanningOptimizations::All, false, false,
+      llvm::sys::toTimeT(std::chrono::system_clock::now()), false);
+  DependencyScanningTool ScanTool(Service, VFS);
+
+  std::vector<std::string> CommandLine0 = {"clang",
+                                           "-target",
+                                           "x86_64-apple-macosx10.7",
+                                           "-c",
+                                           "test0.cpp",
+                                           "-o"
+                                           "test0.cpp.o"};
+  std::vector<std::string> CommandLine1 = {"clang",
+                                           "-target",
+                                           "x86_64-apple-macosx10.7",
+                                           "-c",
+                                           "test1.cpp",
+                                           "-o"
+                                           "test1.cpp.o"};
+
+  std::string Result;
+  ASSERT_THAT_ERROR(
+      ScanTool.getDependencyFile(CommandLine0, CWD).moveInto(Result),
+      llvm::Succeeded());
+
+  VFS->addFile(HeaderPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
+
+  ASSERT_THAT_ERROR(
+      ScanTool.getDependencyFile(CommandLine1, CWD).moveInto(Result),
+      llvm::Succeeded());
+}
diff --git a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
index 7420743c97a2..a68ea72d3816 100644
--- a/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
+++ b/clang/unittests/Tooling/DependencyScanning/DependencyScanningFilesystemTest.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h"
+#include "clang/Tooling/DependencyScanning/DependencyScanningService.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "gtest/gtest.h"
@@ -19,9 +20,10 @@ TEST(DependencyScanningWorkerFilesystem, CacheStatusFailures) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
-  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
 
   DepFS.status("/foo.c");
   EXPECT_EQ(InstrumentingFS->NumStatusCalls, 1u);
@@ -45,9 +47,10 @@ TEST(DependencyScanningFilesystem, CacheGetRealPath) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
-  DependencyScanningWorkerFilesystem DepFS2(SharedCache, InstrumentingFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
+  DependencyScanningWorkerFilesystem DepFS2(Service, InstrumentingFS);
 
   {
     llvm::SmallString<128> Result;
@@ -80,8 +83,9 @@ TEST(DependencyScanningFilesystem, RealPathAndStatusInvariants) {
   InMemoryFS->addFile("/foo.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
   InMemoryFS->addFile("/bar.c", 0, llvm::MemoryBuffer::getMemBuffer(""));
 
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
 
   // Success.
   {
@@ -133,8 +137,9 @@ TEST(DependencyScanningFilesystem, CacheStatOnExists) {
   InMemoryFS->setCurrentWorkingDirectory("/");
   InMemoryFS->addFile("/foo", 0, llvm::MemoryBuffer::getMemBuffer(""));
   InMemoryFS->addFile("/bar", 0, llvm::MemoryBuffer::getMemBuffer(""));
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
 
   DepFS.status("/foo");
   DepFS.status("/foo");
@@ -156,8 +161,9 @@ TEST(DependencyScanningFilesystem, CacheStatFailures) {
   auto InstrumentingFS =
       llvm::makeIntrusiveRefCnt<llvm::vfs::TracingFileSystem>(InMemoryFS);
 
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InstrumentingFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InstrumentingFS);
 
   DepFS.status("/dir");
   DepFS.status("/dir");
@@ -183,8 +189,9 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
   auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
   InMemoryFS->setCurrentWorkingDirectory("/");
 
-  DependencyScanningFilesystemSharedCache SharedCache;
-  DependencyScanningWorkerFilesystem DepFS(SharedCache, InMemoryFS);
+  DependencyScanningService Service(ScanningMode::DependencyDirectivesScan,
+                                    ScanningOutputFormat::Make);
+  DependencyScanningWorkerFilesystem DepFS(Service, InMemoryFS);
 
   bool Path1Exists = DepFS.exists("/path1.suffix");
   EXPECT_EQ(Path1Exists, false);
@@ -197,7 +204,7 @@ TEST(DependencyScanningFilesystem, DiagnoseStaleStatFailures) {
   EXPECT_EQ(Path1Exists, false);
 
   std::vector<llvm::StringRef> InvalidPaths =
-      SharedCache.getInvalidNegativeStatCachedPaths(*InMemoryFS);
+      Service.getSharedCache().getInvalidNegativeStatCachedPaths(*InMemoryFS);
 
   EXPECT_EQ(InvalidPaths.size(), 1u);
   ASSERT_STREQ("/path1.suffix", InvalidPaths[0].str().c_str());
-- 
cgit v1.2.3


From 58f48011b3229b568d3a23a6f4853128712d5f8b Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Fri, 20 Jun 2025 13:28:21 -0700
Subject: [lldb] Add support for x86_64h to scripted process (#145099)

This patch adds support to the haswell sub-architecture (x86_64h) to
scripted processes.

rdar://147208252

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/templates/scripted_process.py                 | 2 +-
 lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/a.out.ips | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lldb/examples/python/templates/scripted_process.py b/lldb/examples/python/templates/scripted_process.py
index c7d5b28b52c0..b6360b851907 100644
--- a/lldb/examples/python/templates/scripted_process.py
+++ b/lldb/examples/python/templates/scripted_process.py
@@ -352,7 +352,7 @@ class ScriptedThread(metaclass=ABCMeta):
     def get_register_info(self):
         if self.register_info is None:
             self.register_info = dict()
-            if self.originating_process.arch == "x86_64":
+            if "x86_64" in self.originating_process.arch:
                 self.register_info["sets"] = ["General Purpose Registers"]
                 self.register_info["registers"] = INTEL64_GPR
             elif (
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/a.out.ips b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/a.out.ips
index cc2f16c01614..a11074f13e90 100644
--- a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/a.out.ips
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/Inputs/a.out.ips
@@ -169,7 +169,7 @@
   },
   {
     "source" : "P",
-    "arch" : "x86_64",
+    "arch" : "x86_64h",
     "base" : 140733734899712,
     "size" : 245760,
     "uuid" : "c5caf30b-0617-3b07-88c7-6319cd06f30a",
-- 
cgit v1.2.3


From b6445ac0c54992993b154875d6afb04eeaa13910 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Fri, 20 Jun 2025 21:37:02 +0100
Subject: [GlobalISel] Create a common register_vector_matchinfo (#144306)

Several combiner use MatchInfo that are just SmallVector<Register>. This
creates a common register_vector_matchinfo that they can all use.
---
 llvm/include/llvm/Target/GlobalISel/Combine.td | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index efd88524a159..4a92dc16c1bf 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -211,6 +211,7 @@ def constantfp_matchinfo : GIDefMatchData<"ConstantFP*">;
 def build_fn_matchinfo :
 GIDefMatchData<"std::function<void(MachineIRBuilder &)>">;
 def unsigned_matchinfo: GIDefMatchData<"unsigned">;
+def register_vector_matchinfo : GIDefMatchData<"SmallVector<Register>">;
 
 def copy_prop : GICombineRule<
   (defs root:$d),
@@ -811,9 +812,8 @@ def zext_trunc_fold: GICombineRule <
   (apply [{ Helper.replaceSingleDefInstWithReg(*${root}, ${matchinfo}); }])
 >;
 
-def not_cmp_fold_matchinfo : GIDefMatchData<"SmallVector<Register, 4>">;
 def not_cmp_fold : GICombineRule<
-  (defs root:$d, not_cmp_fold_matchinfo:$info),
+  (defs root:$d, register_vector_matchinfo:$info),
   (match (wip_match_opcode G_XOR): $d,
   [{ return Helper.matchNotCmp(*${d}, ${info}); }]),
   (apply [{ Helper.applyNotCmp(*${d}, ${info}); }])
@@ -828,9 +828,8 @@ def fneg_fneg_fold: GICombineRule <
 >;
 
 // Fold (unmerge(merge x, y, z)) -> z, y, z.
-def unmerge_merge_matchinfo : GIDefMatchData<"SmallVector<Register, 8>">;
 def unmerge_merge : GICombineRule<
-  (defs root:$d, unmerge_merge_matchinfo:$info),
+  (defs root:$d, register_vector_matchinfo:$info),
   (match (wip_match_opcode G_UNMERGE_VALUES): $d,
   [{ return Helper.matchCombineUnmergeMergeToPlainValues(*${d}, ${info}); }]),
   (apply [{ Helper.applyCombineUnmergeMergeToPlainValues(*${d}, ${info}); }])
@@ -955,9 +954,8 @@ def ptr_add_with_zero: GICombineRule<
          [{ return Helper.matchPtrAddZero(*${root}); }]),
   (apply [{ Helper.applyPtrAddZero(*${root}); }])>;
 
-def regs_small_vec : GIDefMatchData<"SmallVector<Register, 4>">;
 def combine_insert_vec_elts_build_vector : GICombineRule<
-  (defs root:$root, regs_small_vec:$info),
+  (defs root:$root, register_vector_matchinfo:$info),
   (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
     [{ return Helper.matchCombineInsertVecElts(*${root}, ${info}); }]),
   (apply [{ Helper.applyCombineInsertVecElts(*${root}, ${info}); }])>;
@@ -1553,9 +1551,8 @@ def nneg_zext : GICombineRule<
    (apply [{ Helper.applyBuildFnMO(${root}, ${matchinfo}); }])>;
 
 // Combines concat operations
-def concat_matchinfo : GIDefMatchData<"SmallVector<Register>">;
 def combine_concat_vector : GICombineRule<
-  (defs root:$root, concat_matchinfo:$matchinfo),
+  (defs root:$root, register_vector_matchinfo:$matchinfo),
   (match (wip_match_opcode G_CONCAT_VECTORS):$root,
         [{ return Helper.matchCombineConcatVectors(*${root}, ${matchinfo}); }]),
   (apply [{ Helper.applyCombineConcatVectors(*${root}, ${matchinfo}); }])>;
@@ -1567,7 +1564,7 @@ def combine_concat_vector : GICombineRule<
 // ===>
 // c = G_CONCAT_VECTORS x, y, z, undef
 def combine_shuffle_concat : GICombineRule<
-  (defs root:$root, concat_matchinfo:$matchinfo),
+  (defs root:$root, register_vector_matchinfo:$matchinfo),
   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
         [{ return Helper.matchCombineShuffleConcat(*${root}, ${matchinfo}); }]),
   (apply [{ Helper.applyCombineShuffleConcat(*${root}, ${matchinfo}); }])>;
-- 
cgit v1.2.3


From 9c1ce31f546368d6296bc881e9f576ad25c20c73 Mon Sep 17 00:00:00 2001
From: Nishant Patel <nishant.b.patel@intel.com>
Date: Fri, 20 Jun 2025 13:50:25 -0700
Subject: [mlir][vector] Add unroll patterns for vector.load and vector.store
 (#143420)

This PR adds unroll patterns for vector.load and vector.store. This PR is follow up of #137558
---
 mlir/include/mlir/Dialect/Vector/IR/VectorOps.td   |   8 +-
 mlir/lib/Dialect/Vector/IR/VectorOps.cpp           |   8 ++
 .../lib/Dialect/Vector/Transforms/VectorUnroll.cpp | 118 +++++++++++++++++++--
 .../test/Dialect/Vector/vector-unroll-options.mlir |  42 ++++++++
 .../lib/Dialect/Vector/TestVectorTransforms.cpp    |   3 +-
 5 files changed, 170 insertions(+), 9 deletions(-)

diff --git a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
index aef156c5f1d0..926a92eff2eb 100644
--- a/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
+++ b/mlir/include/mlir/Dialect/Vector/IR/VectorOps.td
@@ -1736,7 +1736,9 @@ def Vector_TransferWriteOp :
   let hasVerifier = 1;
 }
 
-def Vector_LoadOp : Vector_Op<"load"> {
+def Vector_LoadOp : Vector_Op<"load", [
+    DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>,
+  ]> {
   let summary = "reads an n-D slice of memory into an n-D vector";
   let description = [{
     The 'vector.load' operation reads an n-D slice of memory into an n-D
@@ -1822,7 +1824,9 @@ def Vector_LoadOp : Vector_Op<"load"> {
       "$base `[` $indices `]` attr-dict `:` type($base) `,` type($result)";
 }
 
-def Vector_StoreOp : Vector_Op<"store"> {
+def Vector_StoreOp : Vector_Op<"store", [
+    DeclareOpInterfaceMethods<VectorUnrollOpInterface, ["getShapeForUnroll"]>,
+  ]> {
   let summary = "writes an n-D vector to an n-D slice of memory";
   let description = [{
     The 'vector.store' operation writes an n-D vector to an n-D slice of memory.
diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
index 6f0ac6bb5828..ee9ab61b670c 100644
--- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
+++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp
@@ -5371,6 +5371,10 @@ OpFoldResult LoadOp::fold(FoldAdaptor) {
   return OpFoldResult();
 }
 
+std::optional<SmallVector<int64_t, 4>> LoadOp::getShapeForUnroll() {
+  return llvm::to_vector<4>(getVectorType().getShape());
+}
+
 //===----------------------------------------------------------------------===//
 // StoreOp
 //===----------------------------------------------------------------------===//
@@ -5406,6 +5410,10 @@ LogicalResult StoreOp::fold(FoldAdaptor adaptor,
   return memref::foldMemRefCast(*this);
 }
 
+std::optional<SmallVector<int64_t, 4>> StoreOp::getShapeForUnroll() {
+  return llvm::to_vector<4>(getVectorType().getShape());
+}
+
 //===----------------------------------------------------------------------===//
 // MaskedLoadOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
index fc443ab0d138..693f4f955994 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorUnroll.cpp
@@ -54,6 +54,28 @@ static SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets,
   return slicedIndices;
 }
 
+// Compute the new indices by adding `offsets` to `originalIndices`.
+// If m < n (m = offsets.size(), n = originalIndices.size()),
+// then only the trailing m values in `originalIndices` are updated.
+static SmallVector<Value> sliceLoadStoreIndices(PatternRewriter &rewriter,
+                                                Location loc,
+                                                OperandRange originalIndices,
+                                                ArrayRef<int64_t> offsets) {
+  assert(offsets.size() <= originalIndices.size() &&
+         "Offsets should not exceed the number of original indices");
+  SmallVector<Value> indices(originalIndices);
+
+  auto start = indices.size() - offsets.size();
+  for (auto [i, offset] : llvm::enumerate(offsets)) {
+    if (offset != 0) {
+      indices[start + i] = rewriter.create<arith::AddIOp>(
+          loc, originalIndices[start + i],
+          rewriter.create<arith::ConstantIndexOp>(loc, offset));
+    }
+  }
+  return indices;
+}
+
 // Clones `op` into a new operations that takes `operands` and returns
 // `resultTypes`.
 static Operation *cloneOpWithOperandsAndTypes(OpBuilder &builder, Location loc,
@@ -631,6 +653,90 @@ private:
   vector::UnrollVectorOptions options;
 };
 
+struct UnrollLoadPattern : public OpRewritePattern<vector::LoadOp> {
+  UnrollLoadPattern(MLIRContext *context,
+                    const vector::UnrollVectorOptions &options,
+                    PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::LoadOp>(context, benefit), options(options) {}
+
+  LogicalResult matchAndRewrite(vector::LoadOp loadOp,
+                                PatternRewriter &rewriter) const override {
+    VectorType vecType = loadOp.getVectorType();
+
+    auto targetShape = getTargetShape(options, loadOp);
+    if (!targetShape)
+      return failure();
+
+    Location loc = loadOp.getLoc();
+    ArrayRef<int64_t> originalShape = vecType.getShape();
+    SmallVector<int64_t> strides(targetShape->size(), 1);
+
+    Value result = rewriter.create<arith::ConstantOp>(
+        loc, vecType, rewriter.getZeroAttr(vecType));
+
+    SmallVector<int64_t> loopOrder =
+        getUnrollOrder(originalShape.size(), loadOp, options);
+
+    auto targetVecType =
+        VectorType::get(*targetShape, vecType.getElementType());
+
+    for (SmallVector<int64_t> offsets :
+         StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
+      SmallVector<Value> indices =
+          sliceLoadStoreIndices(rewriter, loc, loadOp.getIndices(), offsets);
+      Value slicedLoad = rewriter.create<vector::LoadOp>(
+          loc, targetVecType, loadOp.getBase(), indices);
+      result = rewriter.createOrFold<vector::InsertStridedSliceOp>(
+          loc, slicedLoad, result, offsets, strides);
+    }
+    rewriter.replaceOp(loadOp, result);
+    return success();
+  }
+
+private:
+  vector::UnrollVectorOptions options;
+};
+
+struct UnrollStorePattern : public OpRewritePattern<vector::StoreOp> {
+  UnrollStorePattern(MLIRContext *context,
+                     const vector::UnrollVectorOptions &options,
+                     PatternBenefit benefit = 1)
+      : OpRewritePattern<vector::StoreOp>(context, benefit), options(options) {}
+
+  LogicalResult matchAndRewrite(vector::StoreOp storeOp,
+                                PatternRewriter &rewriter) const override {
+    VectorType vecType = storeOp.getVectorType();
+
+    auto targetShape = getTargetShape(options, storeOp);
+    if (!targetShape)
+      return failure();
+
+    Location loc = storeOp.getLoc();
+    ArrayRef<int64_t> originalShape = vecType.getShape();
+    SmallVector<int64_t> strides(targetShape->size(), 1);
+
+    Value base = storeOp.getBase();
+    Value vector = storeOp.getValueToStore();
+
+    SmallVector<int64_t> loopOrder =
+        getUnrollOrder(originalShape.size(), storeOp, options);
+
+    for (SmallVector<int64_t> offsets :
+         StaticTileOffsetRange(originalShape, *targetShape, loopOrder)) {
+      SmallVector<Value> indices =
+          sliceLoadStoreIndices(rewriter, loc, storeOp.getIndices(), offsets);
+      Value slice = rewriter.createOrFold<vector::ExtractStridedSliceOp>(
+          loc, vector, offsets, *targetShape, strides);
+      rewriter.create<vector::StoreOp>(loc, slice, base, indices);
+    }
+    rewriter.eraseOp(storeOp);
+    return success();
+  }
+
+private:
+  vector::UnrollVectorOptions options;
+};
+
 struct UnrollBroadcastPattern : public OpRewritePattern<vector::BroadcastOp> {
   UnrollBroadcastPattern(MLIRContext *context,
                          const vector::UnrollVectorOptions &options,
@@ -699,10 +805,10 @@ private:
 void mlir::vector::populateVectorUnrollPatterns(
     RewritePatternSet &patterns, const UnrollVectorOptions &options,
     PatternBenefit benefit) {
-  patterns
-      .add<UnrollTransferReadPattern, UnrollTransferWritePattern,
-           UnrollContractionPattern, UnrollElementwisePattern,
-           UnrollReductionPattern, UnrollMultiReductionPattern,
-           UnrollTransposePattern, UnrollGatherPattern, UnrollBroadcastPattern>(
-          patterns.getContext(), options, benefit);
+  patterns.add<UnrollTransferReadPattern, UnrollTransferWritePattern,
+               UnrollContractionPattern, UnrollElementwisePattern,
+               UnrollReductionPattern, UnrollMultiReductionPattern,
+               UnrollTransposePattern, UnrollGatherPattern, UnrollLoadPattern,
+               UnrollStorePattern, UnrollBroadcastPattern>(
+      patterns.getContext(), options, benefit);
 }
diff --git a/mlir/test/Dialect/Vector/vector-unroll-options.mlir b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
index fbb178fb49d8..e129cd5c40b9 100644
--- a/mlir/test/Dialect/Vector/vector-unroll-options.mlir
+++ b/mlir/test/Dialect/Vector/vector-unroll-options.mlir
@@ -378,3 +378,45 @@ func.func @vector_broadcast_with_tailing_unit_dim(%v: vector<4x1xf32>) -> vector
 //       CHECK: [[b3:%.+]] = vector.broadcast [[s3]] : vector<2x1xf32> to vector<2x2xf32>
 //       CHECK: [[r3:%.+]] = vector.insert_strided_slice [[b3]], [[r2]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf32> into vector<4x4xf32>
 //       CHECK: return [[r3]] : vector<4x4xf32>
+
+
+func.func @vector_load_2D(%mem: memref<4x4xf16>) -> vector<4x4xf16> {
+  %c0 = arith.constant 0 : index
+  %0 = vector.load %mem[%c0, %c0] : memref<4x4xf16>, vector<4x4xf16>
+  return %0 : vector<4x4xf16>
+}
+
+// CHECK-LABEL: func.func @vector_load_2D(
+// CHECK-SAME:  %[[ARG:.*]]: memref<4x4xf16>) -> vector<4x4xf16> {
+  // CHECK: %[[C2:.*]] = arith.constant 2 : index
+  // CHECK: %[[C0:.*]] = arith.constant 0 : index
+  // CHECK: %[[CST:.*]] = arith.constant dense<0.000000e+00> : vector<4x4xf16>
+  // CHECK: %[[V0:.*]] = vector.load %[[ARG]][%[[C0]], %[[C0]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V1:.*]] = vector.insert_strided_slice %[[V0]], %[[CST]] {offsets = [0, 0], strides = [1, 1]} : vector<2x2xf16> into vector<4x4xf16>
+  // CHECK: %[[V2:.*]] = vector.load %[[ARG]][%[[C0]], %[[C2]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V3:.*]] = vector.insert_strided_slice %[[V2]], %[[V1]] {offsets = [0, 2], strides = [1, 1]} : vector<2x2xf16> into vector<4x4xf16>
+  // CHECK: %[[V4:.*]] = vector.load %[[ARG]][%[[C2]], %[[C0]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V5:.*]] = vector.insert_strided_slice %[[V4]], %[[V3]] {offsets = [2, 0], strides = [1, 1]} : vector<2x2xf16> into vector<4x4xf16>
+  // CHECK: %[[V6:.*]] = vector.load %[[ARG]][%[[C2]], %[[C2]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V7:.*]] = vector.insert_strided_slice %[[V6]], %[[V5]] {offsets = [2, 2], strides = [1, 1]} : vector<2x2xf16> into vector<4x4xf16>
+  // CHECK: return %[[V7]] : vector<4x4xf16>
+
+
+func.func @vector_store_2D(%mem: memref<4x4xf16>, %v: vector<4x4xf16>) {
+  %c0 = arith.constant 0 : index
+  vector.store %v, %mem[%c0, %c0] : memref<4x4xf16>, vector<4x4xf16>
+  return
+}
+
+// CHECK-LABEL: func.func @vector_store_2D(
+// CHECK-SAME:  %[[ARG0:.*]]: memref<4x4xf16>, %[[ARG1:.*]]: vector<4x4xf16>) {
+  // CHECK: %[[C2:.*]] = arith.constant 2 : index
+  // CHECK: %[[C0:.*]] = arith.constant 0 : index
+  // CHECK: %[[V0:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xf16> to vector<2x2xf16>
+  // CHECK: vector.store %[[V0]], %[[ARG0]][%[[C0]], %[[C0]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V1:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [0, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xf16> to vector<2x2xf16>
+  // CHECK: vector.store %[[V1]], %[[ARG0]][%[[C0]], %[[C2]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V2:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 0], sizes = [2, 2], strides = [1, 1]} : vector<4x4xf16> to vector<2x2xf16>
+  // CHECK: vector.store %[[V2]], %[[ARG0]][%[[C2]], %[[C0]]] : memref<4x4xf16>, vector<2x2xf16>
+  // CHECK: %[[V3:.*]] = vector.extract_strided_slice %[[ARG1]] {offsets = [2, 2], sizes = [2, 2], strides = [1, 1]} : vector<4x4xf16> to vector<2x2xf16>
+  // CHECK: vector.store %[[V3]], %[[ARG0]][%[[C2]], %[[C2]]] : memref<4x4xf16>, vector<2x2xf16>
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index 54aa96ba89a0..71000b98fb8f 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -163,7 +163,8 @@ struct TestVectorUnrollingPatterns
             .setFilterConstraint([](Operation *op) {
               return success(
                   isa<arith::AddFOp, vector::FMAOp, vector::MultiDimReductionOp,
-                      vector::BroadcastOp>(op));
+                      vector::BroadcastOp, vector::LoadOp, vector::StoreOp>(
+                      op));
             }));
     populateVectorUnrollPatterns(
         patterns, UnrollVectorOptions()
-- 
cgit v1.2.3


From 4c97a91dc085fba40e5a86c4da8feeffd15b1f8a Mon Sep 17 00:00:00 2001
From: sribee8 <sriya.pratipati@gmail.com>
Date: Fri, 20 Jun 2025 14:00:56 -0700
Subject: [libc] Added closing quote (#145101)

Error message was missing a closing quote, added it.

Co-authored-by: Sriya Pratipati <sriyap@google.com>
---
 libc/hdr/types/mbstate_t.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libc/hdr/types/mbstate_t.h b/libc/hdr/types/mbstate_t.h
index 367c6af7a3ff..d8fadceaaac8 100644
--- a/libc/hdr/types/mbstate_t.h
+++ b/libc/hdr/types/mbstate_t.h
@@ -15,7 +15,7 @@
 
 #else // Overlay mode
 
-#error "Cannot overlay mbstate_t
+#error "Cannot overlay mbstate_t"
 
 #endif // LLVM_LIBC_FULL_BUILD
 
-- 
cgit v1.2.3


From 0c2191b3a79d91d1115b1dbc1a9bd39daed1d9c5 Mon Sep 17 00:00:00 2001
From: Stanislav Mekhanoshin <rampitec@users.noreply.github.com>
Date: Fri, 20 Jun 2025 14:11:29 -0700
Subject: [AMDGPU] Omit image waits in function prologue on gfx1250 (#145097)

---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index f7b88bf2d5eb..a60e2102d4e8 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -2681,6 +2681,10 @@ bool SIInsertWaitcnts::run(MachineFunction &MF) {
         if (CT == LOAD_CNT || CT == DS_CNT || CT == STORE_CNT)
           continue;
 
+        if (!ST->hasImageInsts() &&
+            (CT == EXP_CNT || CT == SAMPLE_CNT || CT == BVH_CNT))
+          continue;
+
         BuildMI(EntryBB, I, DebugLoc(),
                 TII->get(instrsForExtendedCounterTypes[CT]))
             .addImm(0);
-- 
cgit v1.2.3


From 5886f0a18342457f142871ef73f40fc27f936d9b Mon Sep 17 00:00:00 2001
From: Philip Reames <preames@rivosinc.com>
Date: Fri, 20 Jun 2025 14:20:17 -0700
Subject: [RISCV] Allow larger offset when matching build_vector as vid
 sequence (#144756)

I happened to notice that when legalizing get.active.lane.mask with
large vectors we were materializing via constant pool instead of just
shifting by a constant.

We should probably be doing a full cost comparison for the different
lowering strategies as opposed to our current adhoc heuristics, but the
few cases this regresses seem pretty minor. (Given the reduction in vset
toggles, they might not be regressions at all.)

---------

Co-authored-by: Craig Topper <craig.topper@sifive.com>
---
 llvm/lib/Target/RISCV/RISCVISelLowering.cpp        |  10 +-
 llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll    | 130 +++++++++------------
 .../RISCV/rvv/fixed-vectors-fp-buildvec-bf16.ll    |  16 ++-
 .../CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll |  40 ++++---
 .../RISCV/rvv/fixed-vectors-int-buildvec.ll        |   6 +-
 .../RISCV/rvv/fixed-vectors-reduction-int-vp.ll    |  30 ++---
 llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll       |  17 ++-
 7 files changed, 119 insertions(+), 130 deletions(-)

diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 139fa7ba3562..0c54101a1156 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -3724,14 +3724,14 @@ static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
       SplatStepVal = Log2_64(std::abs(StepNumerator));
     }
 
-    // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
-    // threshold since it's the immediate value many RVV instructions accept.
-    // There is no vmul.vi instruction so ensure multiply constant can fit in
-    // a single addi instruction.
+    // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
+    // since it's the immediate value many RVV instructions accept. There is
+    // no vmul.vi instruction so ensure multiply constant can fit in a
+    // single addi instruction.  For the addend, we allow up to 32 bits..
     if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
          (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
         isPowerOf2_32(StepDenominator) &&
-        (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
+        (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
       MVT VIDVT =
           VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
       MVT VIDContainerVT =
diff --git a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll
index ec422a8fbb92..1acc830347de 100644
--- a/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/active_lane_mask.ll
@@ -103,18 +103,16 @@ define <8 x i1> @fv8(ptr %p, i64 %index, i64 %tc) {
 define <32 x i1> @fv32(ptr %p, i64 %index, i64 %tc) {
 ; CHECK-LABEL: fv32:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI8_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI8_0)
 ; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; CHECK-NEXT:    vle8.v v16, (a0)
 ; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    li a0, 16
+; CHECK-NEXT:    vadd.vx v16, v8, a0
 ; CHECK-NEXT:    vsaddu.vx v8, v8, a1
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v24, v16, a2
 ; CHECK-NEXT:    vmsltu.vx v0, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v16
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v16, v8, a2
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf4, ta, ma
-; CHECK-NEXT:    vslideup.vi v0, v16, 2
+; CHECK-NEXT:    vslideup.vi v0, v24, 2
 ; CHECK-NEXT:    ret
   %mask = call <32 x i1> @llvm.get.active.lane.mask.v32i1.i64(i64 %index, i64 %tc)
   ret <32 x i1> %mask
@@ -125,30 +123,24 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) {
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
 ; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    lui a0, %hi(.LCPI9_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI9_0)
-; CHECK-NEXT:    vle8.v v16, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI9_1)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI9_1)
-; CHECK-NEXT:    vle8.v v17, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI9_2)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI9_2)
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vle8.v v18, (a0)
-; CHECK-NEXT:    vmsltu.vx v0, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v16
+; CHECK-NEXT:    li a0, 16
+; CHECK-NEXT:    vsaddu.vx v16, v8, a1
+; CHECK-NEXT:    vmsltu.vx v0, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v24, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v25, v16, a2
+; CHECK-NEXT:    li a0, 48
+; CHECK-NEXT:    vadd.vx v8, v8, a0
 ; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v16, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v17
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v17, v8, a2
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vi v0, v16, 2
+; CHECK-NEXT:    vslideup.vi v0, v24, 2
 ; CHECK-NEXT:    vsetivli zero, 6, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vi v0, v17, 4
+; CHECK-NEXT:    vslideup.vi v0, v25, 4
 ; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; CHECK-NEXT:    vsext.vf8 v8, v18
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
 ; CHECK-NEXT:    vmsltu.vx v16, v8, a2
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
 ; CHECK-NEXT:    vslideup.vi v0, v16, 6
@@ -160,63 +152,49 @@ define <64 x i1> @fv64(ptr %p, i64 %index, i64 %tc) {
 define <128 x i1> @fv128(ptr %p, i64 %index, i64 %tc) {
 ; CHECK-LABEL: fv128:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_0)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_0)
 ; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
-; CHECK-NEXT:    vle8.v v16, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_1)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_1)
-; CHECK-NEXT:    vle8.v v17, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_2)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_2)
-; CHECK-NEXT:    vle8.v v18, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_3)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_3)
 ; CHECK-NEXT:    vid.v v8
-; CHECK-NEXT:    vle8.v v19, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_4)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_4)
-; CHECK-NEXT:    vle8.v v20, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_5)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_5)
-; CHECK-NEXT:    vle8.v v21, (a0)
-; CHECK-NEXT:    lui a0, %hi(.LCPI10_6)
-; CHECK-NEXT:    addi a0, a0, %lo(.LCPI10_6)
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vle8.v v22, (a0)
-; CHECK-NEXT:    vmsltu.vx v0, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v16
+; CHECK-NEXT:    li a0, 80
+; CHECK-NEXT:    vsaddu.vx v16, v8, a1
+; CHECK-NEXT:    vmsltu.vx v0, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 64
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v24, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 96
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v25, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 112
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v26, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 16
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v27, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    li a0, 32
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v28, v16, a2
+; CHECK-NEXT:    vadd.vx v16, v8, a0
+; CHECK-NEXT:    vsaddu.vx v16, v16, a1
+; CHECK-NEXT:    vmsltu.vx v29, v16, a2
+; CHECK-NEXT:    li a0, 48
+; CHECK-NEXT:    vadd.vx v8, v8, a0
 ; CHECK-NEXT:    vsaddu.vx v8, v8, a1
 ; CHECK-NEXT:    vmsltu.vx v16, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v17
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v17, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v18
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v18, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v19
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v19, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v20
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v20, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v21
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v21, v8, a2
-; CHECK-NEXT:    vsext.vf8 v8, v22
-; CHECK-NEXT:    vsaddu.vx v8, v8, a1
-; CHECK-NEXT:    vmsltu.vx v22, v8, a2
 ; CHECK-NEXT:    vsetivli zero, 4, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vi v17, v16, 2
-; CHECK-NEXT:    vslideup.vi v0, v20, 2
+; CHECK-NEXT:    vslideup.vi v25, v24, 2
+; CHECK-NEXT:    vslideup.vi v0, v28, 2
 ; CHECK-NEXT:    vsetivli zero, 6, e8, mf2, tu, ma
-; CHECK-NEXT:    vslideup.vi v17, v18, 4
-; CHECK-NEXT:    vslideup.vi v0, v21, 4
+; CHECK-NEXT:    vslideup.vi v25, v26, 4
+; CHECK-NEXT:    vslideup.vi v0, v29, 4
 ; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vslideup.vi v17, v19, 6
-; CHECK-NEXT:    vslideup.vi v0, v22, 6
+; CHECK-NEXT:    vslideup.vi v25, v27, 6
+; CHECK-NEXT:    vslideup.vi v0, v16, 6
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vslideup.vi v0, v17, 8
+; CHECK-NEXT:    vslideup.vi v0, v25, 8
 ; CHECK-NEXT:    ret
   %mask = call <128 x i1> @llvm.get.active.lane.mask.v128i1.i64(i64 %index, i64 %tc)
   ret <128 x i1> %mask
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec-bf16.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec-bf16.ll
index 3f7cd91737f4..9bb3f4a976ab 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec-bf16.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec-bf16.ll
@@ -94,10 +94,12 @@ define <2 x bfloat> @vid_v2bf16() {
 define <2 x bfloat> @vid_addend1_v2bf16() {
 ; CHECK-LABEL: vid_addend1_v2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, 262148
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    lui a0, 4
+; CHECK-NEXT:    vsll.vi v8, v8, 7
 ; CHECK-NEXT:    addi a0, a0, -128
-; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   ret <2 x bfloat> <bfloat 1.0, bfloat 2.0>
 }
@@ -105,10 +107,12 @@ define <2 x bfloat> @vid_addend1_v2bf16() {
 define <2 x bfloat> @vid_denominator2_v2bf16() {
 ; CHECK-LABEL: vid_denominator2_v2bf16:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a0, 260100
+; CHECK-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    lui a0, 4
+; CHECK-NEXT:    vsll.vi v8, v8, 7
 ; CHECK-NEXT:    addi a0, a0, -256
-; CHECK-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.s.x v8, a0
+; CHECK-NEXT:    vadd.vx v8, v8, a0
 ; CHECK-NEXT:    ret
   ret <2 x bfloat> <bfloat 0.5, bfloat 1.0>
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
index eb40c133514f..564e95c43f68 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-fp-buildvec.ll
@@ -1573,18 +1573,22 @@ define <2 x half> @vid_addend1_v2f16() {
 ;
 ; RV32ZVFHMIN-LABEL: vid_addend1_v2f16:
 ; RV32ZVFHMIN:       # %bb.0:
-; RV32ZVFHMIN-NEXT:    lui a0, 262148
-; RV32ZVFHMIN-NEXT:    addi a0, a0, -1024
-; RV32ZVFHMIN-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV32ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV32ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32ZVFHMIN-NEXT:    vid.v v8
+; RV32ZVFHMIN-NEXT:    li a0, 15
+; RV32ZVFHMIN-NEXT:    vsll.vi v8, v8, 10
+; RV32ZVFHMIN-NEXT:    slli a0, a0, 10
+; RV32ZVFHMIN-NEXT:    vadd.vx v8, v8, a0
 ; RV32ZVFHMIN-NEXT:    ret
 ;
 ; RV64ZVFHMIN-LABEL: vid_addend1_v2f16:
 ; RV64ZVFHMIN:       # %bb.0:
-; RV64ZVFHMIN-NEXT:    lui a0, 262148
-; RV64ZVFHMIN-NEXT:    addi a0, a0, -1024
-; RV64ZVFHMIN-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV64ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64ZVFHMIN-NEXT:    vid.v v8
+; RV64ZVFHMIN-NEXT:    li a0, 15
+; RV64ZVFHMIN-NEXT:    vsll.vi v8, v8, 10
+; RV64ZVFHMIN-NEXT:    slli a0, a0, 10
+; RV64ZVFHMIN-NEXT:    vadd.vx v8, v8, a0
 ; RV64ZVFHMIN-NEXT:    ret
   ret <2 x half> <half 1.0, half 2.0>
 }
@@ -1608,18 +1612,22 @@ define <2 x half> @vid_denominator2_v2f16() {
 ;
 ; RV32ZVFHMIN-LABEL: vid_denominator2_v2f16:
 ; RV32ZVFHMIN:       # %bb.0:
-; RV32ZVFHMIN-NEXT:    lui a0, 245764
-; RV32ZVFHMIN-NEXT:    addi a0, a0, -2048
-; RV32ZVFHMIN-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV32ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV32ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV32ZVFHMIN-NEXT:    vid.v v8
+; RV32ZVFHMIN-NEXT:    li a0, 7
+; RV32ZVFHMIN-NEXT:    vsll.vi v8, v8, 10
+; RV32ZVFHMIN-NEXT:    slli a0, a0, 11
+; RV32ZVFHMIN-NEXT:    vadd.vx v8, v8, a0
 ; RV32ZVFHMIN-NEXT:    ret
 ;
 ; RV64ZVFHMIN-LABEL: vid_denominator2_v2f16:
 ; RV64ZVFHMIN:       # %bb.0:
-; RV64ZVFHMIN-NEXT:    lui a0, 245764
-; RV64ZVFHMIN-NEXT:    addi a0, a0, -2048
-; RV64ZVFHMIN-NEXT:    vsetivli zero, 2, e32, m1, ta, ma
-; RV64ZVFHMIN-NEXT:    vmv.s.x v8, a0
+; RV64ZVFHMIN-NEXT:    vsetivli zero, 2, e16, mf4, ta, ma
+; RV64ZVFHMIN-NEXT:    vid.v v8
+; RV64ZVFHMIN-NEXT:    li a0, 7
+; RV64ZVFHMIN-NEXT:    vsll.vi v8, v8, 10
+; RV64ZVFHMIN-NEXT:    slli a0, a0, 11
+; RV64ZVFHMIN-NEXT:    vadd.vx v8, v8, a0
 ; RV64ZVFHMIN-NEXT:    ret
   ret <2 x half> <half 0.5, half 1.0>
 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
index 1fa96d3c07ca..f235540cc8ff 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-buildvec.ll
@@ -58,10 +58,10 @@ define void @buildvec_vid_plus_imm_v16i8(ptr %x) {
 define void @buildvec_vid_plus_nonimm_v16i8(ptr %x) {
 ; CHECK-LABEL: buildvec_vid_plus_nonimm_v16i8:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    lui a1, %hi(.LCPI4_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI4_0)
 ; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    li a1, 100
+; CHECK-NEXT:    vadd.vx v8, v8, a1
 ; CHECK-NEXT:    vse8.v v8, (a0)
 ; CHECK-NEXT:    ret
   store <16 x i8> <i8 100, i8 101, i8 102, i8 103, i8 104, i8 105, i8 106, i8 107, i8 108, i8 109, i8 110, i8 111, i8 112, i8 113, i8 114, i8 115>, ptr %x
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
index 722a1186faca..dfe8f358b778 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-reduction-int-vp.ll
@@ -1734,19 +1734,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
 ; RV32-NEXT:    .cfi_offset ra, -4
 ; RV32-NEXT:    mv a2, a0
 ; RV32-NEXT:    li a0, 32
-; RV32-NEXT:    lui a3, %hi(.LCPI72_0)
-; RV32-NEXT:    addi a3, a3, %lo(.LCPI72_0)
 ; RV32-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV32-NEXT:    vle8.v v12, (a3)
 ; RV32-NEXT:    vid.v v16
-; RV32-NEXT:    vmsltu.vx v14, v16, a1
-; RV32-NEXT:    li a3, 64
-; RV32-NEXT:    vsext.vf4 v16, v12
 ; RV32-NEXT:    vmsltu.vx v12, v16, a1
+; RV32-NEXT:    vadd.vx v16, v16, a0
+; RV32-NEXT:    vmsltu.vx v13, v16, a1
+; RV32-NEXT:    li a1, 64
 ; RV32-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV32-NEXT:    vslideup.vi v14, v12, 4
-; RV32-NEXT:    vsetvli zero, a3, e8, m4, ta, ma
-; RV32-NEXT:    vmand.mm v0, v14, v0
+; RV32-NEXT:    vslideup.vi v12, v13, 4
+; RV32-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; RV32-NEXT:    vmand.mm v0, v12, v0
 ; RV32-NEXT:    vmv.v.i v12, 1
 ; RV32-NEXT:    vmerge.vvm v8, v12, v8, v0
 ; RV32-NEXT:    vslidedown.vx v12, v8, a0
@@ -1780,19 +1777,16 @@ define signext i8 @vpreduce_mul_v64i8(i8 signext %s, <64 x i8> %v, <64 x i1> %m,
 ; RV64-NEXT:    .cfi_offset ra, -8
 ; RV64-NEXT:    mv a2, a0
 ; RV64-NEXT:    li a0, 32
-; RV64-NEXT:    lui a3, %hi(.LCPI72_0)
-; RV64-NEXT:    addi a3, a3, %lo(.LCPI72_0)
 ; RV64-NEXT:    vsetvli zero, a0, e32, m8, ta, ma
-; RV64-NEXT:    vle8.v v12, (a3)
 ; RV64-NEXT:    vid.v v16
-; RV64-NEXT:    vmsltu.vx v14, v16, a1
-; RV64-NEXT:    li a3, 64
-; RV64-NEXT:    vsext.vf4 v16, v12
 ; RV64-NEXT:    vmsltu.vx v12, v16, a1
+; RV64-NEXT:    vadd.vx v16, v16, a0
+; RV64-NEXT:    vmsltu.vx v13, v16, a1
+; RV64-NEXT:    li a1, 64
 ; RV64-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; RV64-NEXT:    vslideup.vi v14, v12, 4
-; RV64-NEXT:    vsetvli zero, a3, e8, m4, ta, ma
-; RV64-NEXT:    vmand.mm v0, v14, v0
+; RV64-NEXT:    vslideup.vi v12, v13, 4
+; RV64-NEXT:    vsetvli zero, a1, e8, m4, ta, ma
+; RV64-NEXT:    vmand.mm v0, v12, v0
 ; RV64-NEXT:    vmv.v.i v12, 1
 ; RV64-NEXT:    vmerge.vvm v8, v12, v8, v0
 ; RV64-NEXT:    vslidedown.vx v12, v8, a0
diff --git a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll
index bf330ea38a00..c837063231e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vle_vid-vfcvt.ll
@@ -4,10 +4,12 @@
 define void @foo_1(ptr nocapture noundef writeonly %t) {
 ; CHECK-LABEL: foo_1:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lui a1, %hi(.LCPI0_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI0_0)
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    vsll.vi v8, v8, 7
+; CHECK-NEXT:    lui a1, 524288
+; CHECK-NEXT:    vadd.vx v8, v8, a1
+; CHECK-NEXT:    vfcvt.f.x.v v8, v8
 ; CHECK-NEXT:    vse32.v v8, (a0)
 ; CHECK-NEXT:    ret
 entry:
@@ -18,10 +20,13 @@ entry:
 define void @foo_2(ptr nocapture noundef writeonly %t) {
 ; CHECK-LABEL: foo_2:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    lui a1, %hi(.LCPI1_0)
-; CHECK-NEXT:    addi a1, a1, %lo(.LCPI1_0)
 ; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
-; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vid.v v8
+; CHECK-NEXT:    lui a1, 524288
+; CHECK-NEXT:    vsll.vi v8, v8, 7
+; CHECK-NEXT:    addi a1, a1, -512
+; CHECK-NEXT:    vadd.vx v8, v8, a1
+; CHECK-NEXT:    vfcvt.f.x.v v8, v8
 ; CHECK-NEXT:    vse32.v v8, (a0)
 ; CHECK-NEXT:    ret
 entry:
-- 
cgit v1.2.3


From f0d32575a1193741bc9ca90e5beced693cba28b0 Mon Sep 17 00:00:00 2001
From: Amir Ayupov <aaupov@fb.com>
Date: Fri, 20 Jun 2025 14:29:32 -0700
Subject: [BOLT][NFCI] Use FileSymbols for local symbol disambiguation (#89088)

Remove SymbolToFileName mapping from every local symbol to its
containing FILE symbol name, and reuse FileSymbols to disambiguate
local symbols instead.

Also removes the check for `ld-temp.o` file symbol which was added to
prevent LTO build mode from affecting the disambiguated name. This may
cause incompatibility when using the profile collected on a binary built
in a different mode than the input binary.

Addresses #90661.

Speeds up discover file objects by 5-10% for large binaries:
- binary with ~1.2M symbols: 12.6422s -> 12.0297s
- binary with ~4.5M symbols: 48.8851s -> 43.7315s
---
 bolt/lib/Rewrite/RewriteInstance.cpp | 37 ++++++++----------------------------
 1 file changed, 8 insertions(+), 29 deletions(-)

diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index d650e5db54bf..93bd93b6cb98 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -780,14 +780,6 @@ void RewriteInstance::discoverFileObjects() {
 
   // For local symbols we want to keep track of associated FILE symbol name for
   // disambiguation by combined name.
-  StringRef FileSymbolName;
-  bool SeenFileName = false;
-  struct SymbolRefHash {
-    size_t operator()(SymbolRef const &S) const {
-      return std::hash<decltype(DataRefImpl::p)>{}(S.getRawDataRefImpl().p);
-    }
-  };
-  std::unordered_map<SymbolRef, StringRef, SymbolRefHash> SymbolToFileName;
   for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
     Expected<StringRef> NameOrError = Symbol.getName();
     if (NameOrError && NameOrError->starts_with("__asan_init")) {
@@ -806,21 +798,8 @@ void RewriteInstance::discoverFileObjects() {
     if (cantFail(Symbol.getFlags()) & SymbolRef::SF_Undefined)
       continue;
 
-    if (cantFail(Symbol.getType()) == SymbolRef::ST_File) {
+    if (cantFail(Symbol.getType()) == SymbolRef::ST_File)
       FileSymbols.emplace_back(Symbol);
-      StringRef Name =
-          cantFail(std::move(NameOrError), "cannot get symbol name for file");
-      // Ignore Clang LTO artificial FILE symbol as it is not always generated,
-      // and this uncertainty is causing havoc in function name matching.
-      if (Name == "ld-temp.o")
-        continue;
-      FileSymbolName = Name;
-      SeenFileName = true;
-      continue;
-    }
-    if (!FileSymbolName.empty() &&
-        !(cantFail(Symbol.getFlags()) & SymbolRef::SF_Global))
-      SymbolToFileName[Symbol] = FileSymbolName;
   }
 
   // Sort symbols in the file by value. Ignore symbols from non-allocatable
@@ -1028,14 +1007,14 @@ void RewriteInstance::discoverFileObjects() {
       // The <id> field is used for disambiguation of local symbols since there
       // could be identical function names coming from identical file names
       // (e.g. from different directories).
-      std::string AltPrefix;
-      auto SFI = SymbolToFileName.find(Symbol);
-      if (SymbolType == SymbolRef::ST_Function && SFI != SymbolToFileName.end())
-        AltPrefix = Name + "/" + std::string(SFI->second);
+      auto SFI = llvm::upper_bound(FileSymbols, ELFSymbolRef(Symbol));
+      if (SymbolType == SymbolRef::ST_Function && SFI != FileSymbols.begin()) {
+        StringRef FileSymbolName = cantFail(SFI[-1].getName());
+        if (!FileSymbolName.empty())
+          AlternativeName = NR.uniquify(Name + "/" + FileSymbolName.str());
+      }
 
       UniqueName = NR.uniquify(Name);
-      if (!AltPrefix.empty())
-        AlternativeName = NR.uniquify(AltPrefix);
     }
 
     uint64_t SymbolSize = ELFSymbolRef(Symbol).getSize();
@@ -1294,7 +1273,7 @@ void RewriteInstance::discoverFileObjects() {
                              FDE->getAddressRange());
   }
 
-  BC->setHasSymbolsWithFileName(SeenFileName);
+  BC->setHasSymbolsWithFileName(FileSymbols.size());
 
   // Now that all the functions were created - adjust their boundaries.
   adjustFunctionBoundaries();
-- 
cgit v1.2.3


From a91154343780dae022bb314aa76f0b0affc28b62 Mon Sep 17 00:00:00 2001
From: Uzair Nawaz <uzairnawaz@google.com>
Date: Fri, 20 Jun 2025 21:43:00 +0000
Subject: [libc] Implemented wcrtomb internal function and public libc function
 (#144596)

Implemented internal wcrtomb function using the CharacterConverter class
public libc function calls this internal function to perform the
conversion
---
 libc/config/linux/x86_64/entrypoints.txt |  1 +
 libc/include/wchar.yaml                  |  8 +++
 libc/src/__support/wchar/CMakeLists.txt  | 16 ++++++
 libc/src/__support/wchar/wcrtomb.cpp     | 49 +++++++++++++++++
 libc/src/__support/wchar/wcrtomb.h       | 26 +++++++++
 libc/src/wchar/CMakeLists.txt            | 14 +++++
 libc/src/wchar/wcrtomb.cpp               | 45 ++++++++++++++++
 libc/src/wchar/wcrtomb.h                 | 23 ++++++++
 libc/test/src/wchar/CMakeLists.txt       | 14 +++++
 libc/test/src/wchar/wcrtomb_test.cpp     | 93 ++++++++++++++++++++++++++++++++
 10 files changed, 289 insertions(+)
 create mode 100644 libc/src/__support/wchar/wcrtomb.cpp
 create mode 100644 libc/src/__support/wchar/wcrtomb.h
 create mode 100644 libc/src/wchar/wcrtomb.cpp
 create mode 100644 libc/src/wchar/wcrtomb.h
 create mode 100644 libc/test/src/wchar/wcrtomb_test.cpp

diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 8bf6c402b039..4d94f10196fd 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -1247,6 +1247,7 @@ if(LLVM_LIBC_FULL_BUILD)
 
     # wchar.h entrypoints
     libc.src.wchar.mbrtowc
+    libc.src.wchar.wcrtomb
   )
 endif()
 
diff --git a/libc/include/wchar.yaml b/libc/include/wchar.yaml
index c036636e12c3..64eb38171066 100644
--- a/libc/include/wchar.yaml
+++ b/libc/include/wchar.yaml
@@ -159,6 +159,14 @@ functions:
       - type: wchar_t *__restrict
       - type: const wchar_t *__restrict
       - type: size_t
+  - name: wcrtomb
+    standards:
+      - stdc
+    return_type: size_t
+    arguments:
+      - type: char *__restrict
+      - type: wchar_t
+      - type: mbstate_t *__restrict
   - name: wcscpy
     standards:
       - stdc
diff --git a/libc/src/__support/wchar/CMakeLists.txt b/libc/src/__support/wchar/CMakeLists.txt
index 479c1dff2c6e..6aade4ccc84a 100644
--- a/libc/src/__support/wchar/CMakeLists.txt
+++ b/libc/src/__support/wchar/CMakeLists.txt
@@ -20,6 +20,22 @@ add_object_library(
     .mbstate
 )
 
+add_object_library(
+  wcrtomb
+  HDRS
+    wcrtomb.h
+  SRCS 
+    wcrtomb.cpp
+  DEPENDS
+    libc.hdr.types.char32_t
+    libc.hdr.types.size_t
+    libc.hdr.types.wchar_t
+    libc.src.__support.error_or
+    libc.src.__support.common
+    .character_converter
+    .mbstate
+)
+
 add_object_library(
   mbrtowc
   HDRS
diff --git a/libc/src/__support/wchar/wcrtomb.cpp b/libc/src/__support/wchar/wcrtomb.cpp
new file mode 100644
index 000000000000..8ca3d17ad6ce
--- /dev/null
+++ b/libc/src/__support/wchar/wcrtomb.cpp
@@ -0,0 +1,49 @@
+//===-- Implementation of wcrtomb -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/__support/wchar/wcrtomb.h"
+#include "src/__support/error_or.h"
+#include "src/__support/wchar/character_converter.h"
+#include "src/__support/wchar/mbstate.h"
+
+#include "hdr/types/char32_t.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_assert.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+ErrorOr<size_t> wcrtomb(char *__restrict s, wchar_t wc,
+                        mbstate *__restrict ps) {
+  static_assert(sizeof(wchar_t) == 4);
+
+  CharacterConverter cr(ps);
+
+  if (s == nullptr)
+    return Error(-1);
+
+  int status = cr.push(static_cast<char32_t>(wc));
+  if (status != 0)
+    return Error(status);
+
+  size_t count = 0;
+  while (!cr.isEmpty()) {
+    auto utf8 = cr.pop_utf8(); // can never fail as long as the push succeeded
+    LIBC_ASSERT(utf8.has_value());
+
+    *s = utf8.value();
+    s++;
+    count++;
+  }
+  return count;
+}
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/__support/wchar/wcrtomb.h b/libc/src/__support/wchar/wcrtomb.h
new file mode 100644
index 000000000000..bcd39a92a3b7
--- /dev/null
+++ b/libc/src/__support/wchar/wcrtomb.h
@@ -0,0 +1,26 @@
+//===-- Implementation header for wcrtomb ---------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
+#define LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
+
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/error_or.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/mbstate.h"
+
+namespace LIBC_NAMESPACE_DECL {
+namespace internal {
+
+ErrorOr<size_t> wcrtomb(char *__restrict s, wchar_t wc, mbstate *__restrict ps);
+
+} // namespace internal
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC__SUPPORT_WCHAR_WCRTOMB_H
diff --git a/libc/src/wchar/CMakeLists.txt b/libc/src/wchar/CMakeLists.txt
index 163c29847e6a..ec33caccb16d 100644
--- a/libc/src/wchar/CMakeLists.txt
+++ b/libc/src/wchar/CMakeLists.txt
@@ -34,6 +34,20 @@ add_entrypoint_object(
     libc.src.__support.wctype_utils
 )
 
+add_entrypoint_object(
+  wcrtomb
+  SRCS
+    wcrtomb.cpp
+  HDRS
+    wcrtomb.h
+  DEPENDS
+    libc.hdr.types.wchar_t
+    libc.hdr.types.mbstate_t
+    libc.src.__support.libc_errno
+    libc.src.__support.wchar.wcrtomb
+    libc.src.__support.wchar.mbstate
+)
+
 add_entrypoint_object(
   mbrtowc
   SRCS
diff --git a/libc/src/wchar/wcrtomb.cpp b/libc/src/wchar/wcrtomb.cpp
new file mode 100644
index 000000000000..6d604a00599e
--- /dev/null
+++ b/libc/src/wchar/wcrtomb.cpp
@@ -0,0 +1,45 @@
+//===-- Implementation of wcrtomb -----------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/wchar/wcrtomb.h"
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/common.h"
+#include "src/__support/libc_errno.h"
+#include "src/__support/macros/config.h"
+#include "src/__support/wchar/mbstate.h"
+#include "src/__support/wchar/wcrtomb.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+LLVM_LIBC_FUNCTION(size_t, wcrtomb,
+                   (char *__restrict s, wchar_t wc, mbstate_t *__restrict ps)) {
+  static internal::mbstate internal_mbstate;
+
+  // when s is nullptr, this is equivalent to wcrtomb(buf, L'\0', ps)
+  char buf[sizeof(wchar_t) / sizeof(char)];
+  if (s == nullptr) {
+    s = buf;
+    wc = L'\0';
+  }
+
+  auto result = internal::wcrtomb(
+      s, wc,
+      ps == nullptr ? &internal_mbstate
+                    : reinterpret_cast<internal::mbstate *>(ps));
+
+  if (!result.has_value()) {
+    libc_errno = EILSEQ;
+    return -1;
+  }
+
+  return result.value();
+}
+
+} // namespace LIBC_NAMESPACE_DECL
diff --git a/libc/src/wchar/wcrtomb.h b/libc/src/wchar/wcrtomb.h
new file mode 100644
index 000000000000..06c42f158122
--- /dev/null
+++ b/libc/src/wchar/wcrtomb.h
@@ -0,0 +1,23 @@
+//===-- Implementation header for wcrtomb -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_WCHAR_WCRTOMB_H
+#define LLVM_LIBC_SRC_WCHAR_WCRTOMB_H
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/size_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/macros/config.h"
+
+namespace LIBC_NAMESPACE_DECL {
+
+size_t wcrtomb(char *__restrict s, wchar_t wc, mbstate_t *__restrict ps);
+
+} // namespace LIBC_NAMESPACE_DECL
+
+#endif // LLVM_LIBC_SRC_WCHAR_WCRTOMB_H
diff --git a/libc/test/src/wchar/CMakeLists.txt b/libc/test/src/wchar/CMakeLists.txt
index d4cae1f6228b..184e482c895b 100644
--- a/libc/test/src/wchar/CMakeLists.txt
+++ b/libc/test/src/wchar/CMakeLists.txt
@@ -47,6 +47,20 @@ add_libc_test(
     libc.src.wchar.wctob
 )
 
+add_libc_test(
+  wcrtomb_test
+  SUITE
+    libc_wchar_unittests
+  SRCS
+    wcrtomb_test.cpp
+  DEPENDS
+    libc.src.wchar.wcrtomb
+    libc.src.string.memset
+    libc.hdr.types.wchar_t
+    libc.hdr.types.mbstate_t
+    libc.src.__support.libc_errno
+)
+
 add_libc_test(
   wmemset_test 
   SUITE
diff --git a/libc/test/src/wchar/wcrtomb_test.cpp b/libc/test/src/wchar/wcrtomb_test.cpp
new file mode 100644
index 000000000000..c06b39ae0143
--- /dev/null
+++ b/libc/test/src/wchar/wcrtomb_test.cpp
@@ -0,0 +1,93 @@
+//===-- Unittests for wcrtomb --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "hdr/types/mbstate_t.h"
+#include "hdr/types/wchar_t.h"
+#include "src/__support/libc_errno.h"
+#include "src/string/memset.h"
+#include "src/wchar/wcrtomb.h"
+#include "test/UnitTest/Test.h"
+
+TEST(LlvmLibcWCRToMBTest, OneByte) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  wchar_t wc = L'U';
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+  ASSERT_EQ(cnt, static_cast<size_t>(1));
+  ASSERT_EQ(mb[0], 'U');
+}
+
+TEST(LlvmLibcWCRToMBTest, TwoByte) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  // testing utf32: 0xff -> utf8: 0xc3 0xbf
+  wchar_t wc = 0xff;
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+  ASSERT_EQ(cnt, static_cast<size_t>(2));
+  ASSERT_EQ(mb[0], static_cast<char>(0xc3));
+  ASSERT_EQ(mb[1], static_cast<char>(0xbf));
+}
+
+TEST(LlvmLibcWCRToMBTest, ThreeByte) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  // testing utf32: 0xac15 -> utf8: 0xea 0xb0 0x95
+  wchar_t wc = 0xac15;
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+  ASSERT_EQ(cnt, static_cast<size_t>(3));
+  ASSERT_EQ(mb[0], static_cast<char>(0xea));
+  ASSERT_EQ(mb[1], static_cast<char>(0xb0));
+  ASSERT_EQ(mb[2], static_cast<char>(0x95));
+}
+
+TEST(LlvmLibcWCRToMBTest, FourByte) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  // testing utf32: 0x1f921 -> utf8: 0xf0 0x9f 0xa4 0xa1
+  wchar_t wc = 0x1f921;
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+  ASSERT_EQ(cnt, static_cast<size_t>(4));
+  ASSERT_EQ(mb[0], static_cast<char>(0xf0));
+  ASSERT_EQ(mb[1], static_cast<char>(0x9f));
+  ASSERT_EQ(mb[2], static_cast<char>(0xa4));
+  ASSERT_EQ(mb[3], static_cast<char>(0xa1));
+}
+
+TEST(LlvmLibcWCRToMBTest, NullString) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  wchar_t wc = L'A';
+  char mb[4];
+
+  // should be equivalent to the call wcrtomb(buf, L'\0', state)
+  size_t cnt1 = LIBC_NAMESPACE::wcrtomb(nullptr, wc, &state);
+  size_t cnt2 = LIBC_NAMESPACE::wcrtomb(mb, L'\0', &state);
+
+  ASSERT_EQ(cnt1, cnt2);
+}
+
+TEST(LlvmLibcWCRToMBTest, NullState) {
+  wchar_t wc = L'A';
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, nullptr);
+  ASSERT_EQ(cnt, static_cast<size_t>(1));
+}
+
+TEST(LlvmLibcWCRToMBTest, InvalidWchar) {
+  mbstate_t state;
+  LIBC_NAMESPACE::memset(&state, 0, sizeof(mbstate_t));
+  wchar_t wc = 0x12ffff;
+  char mb[4];
+  size_t cnt = LIBC_NAMESPACE::wcrtomb(mb, wc, &state);
+  ASSERT_EQ(cnt, static_cast<size_t>(-1));
+  ASSERT_EQ(static_cast<int>(libc_errno), EILSEQ);
+}
-- 
cgit v1.2.3


From e6ee2c7c7b36825331b39e221725780167457e6d Mon Sep 17 00:00:00 2001
From: Finn Plummer <finn.c.plum@gmail.com>
Date: Fri, 20 Jun 2025 14:54:58 -0700
Subject: [HLSL][RootSignature] Implement validation of resource ranges for
 `RootDescriptors` (#140962)

As was established
[previously](https://github.com/llvm/llvm-project/pull/140957), we
created a structure to model a resource range and to detect an overlap
in a given set of these.

However, a resource range only overlaps with another resource range if
they have:
- equivalent ResourceClass (SRV, UAV, CBuffer, Sampler)
- equivalent resource name-space
- overlapping shader visibility

For instance, the following don't overlap even though they have the same
register range:
- `CBV(b0)` and `SRV(t0)` (different resource class)
- `CBV(b0, space = 0)` and `CBV(b0, space = 1)` (different space)
- `CBV(b0, visibility = Pixel)` and `CBV(b0, visibility = Domain)`
(non-overlapping visibility)

The first two clauses are naturally modelled by grouping all the
`RangeInfo`s that have the equivalent `ResourceClass` and `Space` values
together and check if there is any overlap on a `ResourceRange` for all
these `RangeInfo`s. However, `Visibility` is not quite as easily mapped
(`Visibility = All` would overlap with any other visibility). So we will
instead need to track a `ResourceRange` for each of the `Visibility`
types in a group. Then we can determine when inserting a range of the
same group if it would overlap with any overlapping visibilities.

The collection of `RangeInfo` for `RootDescriptor`s, sorting of the
`RangeInfo`s into the groups and finally the insertion of each point
into their respective `ResourceRange`s are implemented. Furthermore, we
integrate this into `SemaHLSL` to provide a diagnostic for each entry
function that uses the invalid root signature.

- Implements collection of `RangeInfo` for `RootDescriptors`
- Implements resource range validation in `SemaHLSL`
- Add diagnostic testing of error production in
`RootSignature-resource-ranges-err.hlsl`
- Add testing to ensure no errors are raised in valid root signatures
`RootSignature-resource-ranges.hlsl`

Part 2 of https://github.com/llvm/llvm-project/issues/129942

A final pr will be produced to integrate the analysis of
`DescriptorTable`, `StaticSampler` and `RootConstants` by defining how
to construct the `RangeInfo` from their elements respectively.
---
 clang/include/clang/Basic/DiagnosticSemaKinds.td   |   5 +
 clang/include/clang/Sema/SemaHLSL.h                |   2 +
 clang/lib/Sema/SemaHLSL.cpp                        | 130 ++++++++++++++++++++-
 .../RootSignature-resource-ranges-err.hlsl         |  21 ++++
 .../SemaHLSL/RootSignature-resource-ranges.hlsl    |  15 +++
 .../llvm/Frontend/HLSL/HLSLRootSignatureUtils.h    |  11 +-
 llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp  |   2 +
 7 files changed, 183 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
 create mode 100644 clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 34b798a09c21..f2f2152b8bbb 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -13054,6 +13054,11 @@ def err_invalid_hlsl_resource_type: Error<
 def err_hlsl_spirv_only: Error<"%0 is only available for the SPIR-V target">;
 def err_hlsl_vk_literal_must_contain_constant: Error<"the argument to vk::Literal must be a vk::integral_constant">;
 
+def err_hlsl_resource_range_overlap: Error<
+  "resource ranges %select{t|u|b|s}0[%1;%2] and %select{t|u|b|s}3[%4;%5] "
+  "overlap within space = %6 and visibility = "
+  "%select{All|Vertex|Hull|Domain|Geometry|Pixel|Amplification|Mesh}7">;
+
 // Layout randomization diagnostics.
 def err_non_designated_init_used : Error<
   "a randomized struct can only be initialized with a designated initializer">;
diff --git a/clang/include/clang/Sema/SemaHLSL.h b/clang/include/clang/Sema/SemaHLSL.h
index 97091792ba23..7d7eae4db532 100644
--- a/clang/include/clang/Sema/SemaHLSL.h
+++ b/clang/include/clang/Sema/SemaHLSL.h
@@ -134,6 +134,8 @@ public:
       SourceLocation Loc, IdentifierInfo *DeclIdent,
       SmallVector<llvm::hlsl::rootsig::RootElement> &Elements);
 
+  // Returns true when D is invalid and a diagnostic was produced
+  bool handleRootSignatureDecl(HLSLRootSignatureDecl *D, SourceLocation Loc);
   void handleRootSignatureAttr(Decl *D, const ParsedAttr &AL);
   void handleNumThreadsAttr(Decl *D, const ParsedAttr &AL);
   void handleWaveSizeAttr(Decl *D, const ParsedAttr &AL);
diff --git a/clang/lib/Sema/SemaHLSL.cpp b/clang/lib/Sema/SemaHLSL.cpp
index 9b43ee00810b..d003967a522a 100644
--- a/clang/lib/Sema/SemaHLSL.cpp
+++ b/clang/lib/Sema/SemaHLSL.cpp
@@ -39,6 +39,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Frontend/HLSL/HLSLRootSignatureUtils.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/DXILABI.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -1068,10 +1069,138 @@ void SemaHLSL::ActOnFinishRootSignatureDecl(
       SemaRef.getASTContext(), /*DeclContext=*/SemaRef.CurContext, Loc,
       DeclIdent, Elements);
 
+  if (handleRootSignatureDecl(SignatureDecl, Loc))
+    return;
+
   SignatureDecl->setImplicit();
   SemaRef.PushOnScopeChains(SignatureDecl, SemaRef.getCurScope());
 }
 
+bool SemaHLSL::handleRootSignatureDecl(HLSLRootSignatureDecl *D,
+                                       SourceLocation Loc) {
+  // The following conducts analysis on resource ranges to detect and report
+  // any overlaps in resource ranges.
+  //
+  // A resource range overlaps with another resource range if they have:
+  // - equivalent ResourceClass (SRV, UAV, CBuffer, Sampler)
+  // - equivalent resource space
+  // - overlapping visbility
+  //
+  // The following algorithm is implemented in the following steps:
+  //
+  // 1. Collect RangeInfo from relevant RootElements:
+  //   - RangeInfo will retain the interval, ResourceClass, Space and Visibility
+  // 2. Sort the RangeInfo's such that they are grouped together by
+  //  ResourceClass and Space (GroupT defined below)
+  // 3. Iterate through the collected RangeInfos by their groups
+  //   - For each group we will have a ResourceRange for each visibility
+  //   - As we iterate through we will:
+  //      A: Insert the current RangeInfo into the corresponding Visibility
+  //   ResourceRange
+  //      B: Check for overlap with any overlapping Visibility ResourceRange
+  using RangeInfo = llvm::hlsl::rootsig::RangeInfo;
+  using ResourceRange = llvm::hlsl::rootsig::ResourceRange;
+  using GroupT = std::pair<ResourceClass, /*Space*/ uint32_t>;
+
+  // 1. Collect RangeInfos
+  llvm::SmallVector<RangeInfo> Infos;
+  for (const llvm::hlsl::rootsig::RootElement &Elem : D->getRootElements()) {
+    if (const auto *Descriptor =
+            std::get_if<llvm::hlsl::rootsig::RootDescriptor>(&Elem)) {
+      RangeInfo Info;
+      Info.LowerBound = Descriptor->Reg.Number;
+      Info.UpperBound = Info.LowerBound; // use inclusive ranges []
+
+      Info.Class =
+          llvm::dxil::ResourceClass(llvm::to_underlying(Descriptor->Type));
+      Info.Space = Descriptor->Space;
+      Info.Visibility = Descriptor->Visibility;
+      Infos.push_back(Info);
+    }
+  }
+
+  // 2. Sort the RangeInfo's by their GroupT to form groupings
+  std::sort(Infos.begin(), Infos.end(), [](RangeInfo A, RangeInfo B) {
+    return std::tie(A.Class, A.Space) < std::tie(B.Class, B.Space);
+  });
+
+  // 3. First we will init our state to track:
+  if (Infos.size() == 0)
+    return false; // No ranges to overlap
+  GroupT CurGroup = {Infos[0].Class, Infos[0].Space};
+  bool HadOverlap = false;
+
+  // Create a ResourceRange for each Visibility
+  ResourceRange::MapT::Allocator Allocator;
+  std::array<ResourceRange, 8> Ranges = {
+      ResourceRange(Allocator), // All
+      ResourceRange(Allocator), // Vertex
+      ResourceRange(Allocator), // Hull
+      ResourceRange(Allocator), // Domain
+      ResourceRange(Allocator), // Geometry
+      ResourceRange(Allocator), // Pixel
+      ResourceRange(Allocator), // Amplification
+      ResourceRange(Allocator), // Mesh
+  };
+
+  // Reset the ResourceRanges for when we iterate through a new group
+  auto ClearRanges = [&Ranges]() {
+    for (ResourceRange &Range : Ranges)
+      Range.clear();
+  };
+
+  // Helper to report diagnostics
+  auto ReportOverlap = [this, Loc, &HadOverlap](const RangeInfo *Info,
+                                                const RangeInfo *OInfo) {
+    HadOverlap = true;
+    auto CommonVis =
+        Info->Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
+            ? OInfo->Visibility
+            : Info->Visibility;
+    this->Diag(Loc, diag::err_hlsl_resource_range_overlap)
+        << llvm::to_underlying(Info->Class) << Info->LowerBound
+        << Info->UpperBound << llvm::to_underlying(OInfo->Class)
+        << OInfo->LowerBound << OInfo->UpperBound << Info->Space << CommonVis;
+  };
+
+  // 3: Iterate through collected RangeInfos
+  for (const RangeInfo &Info : Infos) {
+    GroupT InfoGroup = {Info.Class, Info.Space};
+    // Reset our ResourceRanges when we enter a new group
+    if (CurGroup != InfoGroup) {
+      ClearRanges();
+      CurGroup = InfoGroup;
+    }
+
+    // 3A: Insert range info into corresponding Visibility ResourceRange
+    ResourceRange &VisRange = Ranges[llvm::to_underlying(Info.Visibility)];
+    if (std::optional<const RangeInfo *> Overlapping = VisRange.insert(Info))
+      ReportOverlap(&Info, Overlapping.value());
+
+    // 3B: Check for overlap in all overlapping Visibility ResourceRanges
+    //
+    // If the range that we are inserting has ShaderVisiblity::All it needs to
+    // check for an overlap in all other visibility types as well.
+    // Otherwise, the range that is inserted needs to check that it does not
+    // overlap with ShaderVisibility::All.
+    //
+    // OverlapRanges will be an ArrayRef to all non-all visibility
+    // ResourceRanges in the former case and it will be an ArrayRef to just the
+    // all visiblity ResourceRange in the latter case.
+    ArrayRef<ResourceRange> OverlapRanges =
+        Info.Visibility == llvm::hlsl::rootsig::ShaderVisibility::All
+            ? ArrayRef<ResourceRange>{Ranges}.drop_front()
+            : ArrayRef<ResourceRange>{Ranges}.take_front();
+
+    for (const ResourceRange &Range : OverlapRanges)
+      if (std::optional<const RangeInfo *> Overlapping =
+              Range.getOverlapping(Info))
+        ReportOverlap(&Info, Overlapping.value());
+  }
+
+  return HadOverlap;
+}
+
 void SemaHLSL::handleRootSignatureAttr(Decl *D, const ParsedAttr &AL) {
   if (AL.getNumArgs() != 1) {
     Diag(AL.getLoc(), diag::err_attribute_wrong_number_arguments) << AL << 1;
@@ -1093,7 +1222,6 @@ void SemaHLSL::handleRootSignatureAttr(Decl *D, const ParsedAttr &AL) {
   if (SemaRef.LookupQualifiedName(R, D->getDeclContext()))
     if (auto *SignatureDecl =
             dyn_cast<HLSLRootSignatureDecl>(R.getFoundDecl())) {
-      // Perform validation of constructs here
       D->addAttr(::new (getASTContext()) RootSignatureAttr(
           getASTContext(), AL, Ident, SignatureDecl));
     }
diff --git a/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl b/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
new file mode 100644
index 000000000000..e5152e72d480
--- /dev/null
+++ b/clang/test/SemaHLSL/RootSignature-resource-ranges-err.hlsl
@@ -0,0 +1,21 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - %s -verify
+
+// expected-error@+1 {{resource ranges b[42;42] and b[42;42] overlap within space = 0 and visibility = All}}
+[RootSignature("CBV(b42), CBV(b42)")]
+void bad_root_signature_0() {}
+
+// expected-error@+1 {{resource ranges t[0;0] and t[0;0] overlap within space = 3 and visibility = All}}
+[RootSignature("SRV(t0, space = 3), SRV(t0, space = 3)")]
+void bad_root_signature_1() {}
+
+// expected-error@+1 {{resource ranges u[0;0] and u[0;0] overlap within space = 0 and visibility = Pixel}}
+[RootSignature("UAV(u0, visibility = SHADER_VISIBILITY_PIXEL), UAV(u0, visibility = SHADER_VISIBILITY_PIXEL)")]
+void bad_root_signature_2() {}
+
+// expected-error@+1 {{resource ranges u[0;0] and u[0;0] overlap within space = 0 and visibility = Pixel}}
+[RootSignature("UAV(u0, visibility = SHADER_VISIBILITY_ALL), UAV(u0, visibility = SHADER_VISIBILITY_PIXEL)")]
+void bad_root_signature_3() {}
+
+// expected-error@+1 {{resource ranges u[0;0] and u[0;0] overlap within space = 0 and visibility = Pixel}}
+[RootSignature("UAV(u0, visibility = SHADER_VISIBILITY_PIXEL), UAV(u0, visibility = SHADER_VISIBILITY_ALL)")]
+void bad_root_signature_4() {}
diff --git a/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl b/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl
new file mode 100644
index 000000000000..5778fb2ae4eb
--- /dev/null
+++ b/clang/test/SemaHLSL/RootSignature-resource-ranges.hlsl
@@ -0,0 +1,15 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -x hlsl -o - %s -verify
+
+// expected-no-diagnostics
+
+[RootSignature("CBV(b0), CBV(b1)")]
+void valid_root_signature_0() {}
+
+[RootSignature("CBV(b0, visibility = SHADER_VISIBILITY_DOMAIN), CBV(b0, visibility = SHADER_VISIBILITY_PIXEL)")]
+void valid_root_signature_1() {}
+
+[RootSignature("CBV(b0, space = 1), CBV(b0, space = 2)")]
+void valid_root_signature_2() {}
+
+[RootSignature("CBV(b0), SRV(t0)")]
+void valid_root_signature_3() {}
diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
index 25c2a9f0cc80..4769fd055996 100644
--- a/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
+++ b/llvm/include/llvm/Frontend/HLSL/HLSLRootSignatureUtils.h
@@ -71,13 +71,17 @@ private:
   SmallVector<Metadata *> GeneratedMetadata;
 };
 
-// RangeInfo holds the information to correctly construct a ResourceRange
-// and retains this information to be used for displaying a better diagnostic
 struct RangeInfo {
   const static uint32_t Unbounded = ~0u;
 
+  // Interval information
   uint32_t LowerBound;
   uint32_t UpperBound;
+
+  // Information retained for diagnostics
+  llvm::dxil::ResourceClass Class;
+  uint32_t Space;
+  ShaderVisibility Visibility;
 };
 
 class ResourceRange {
@@ -98,6 +102,9 @@ public:
   // Return the mapped RangeInfo at X or nullptr if no mapping exists
   const RangeInfo *lookup(uint32_t X) const;
 
+  // Removes all entries of the ResourceRange
+  void clear();
+
   // Insert the required (sub-)intervals such that the interval of [a;b] =
   // [Info.LowerBound, Info.UpperBound] is covered and points to a valid
   // RangeInfo &.
diff --git a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp b/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
index a1ddb318055b..f95c141c54d8 100644
--- a/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
+++ b/llvm/lib/Frontend/HLSL/HLSLRootSignatureUtils.cpp
@@ -509,6 +509,8 @@ const RangeInfo *ResourceRange::lookup(uint32_t X) const {
   return Intervals.lookup(X, nullptr);
 }
 
+void ResourceRange::clear() { return Intervals.clear(); }
+
 std::optional<const RangeInfo *> ResourceRange::insert(const RangeInfo &Info) {
   uint32_t LowerBound = Info.LowerBound;
   uint32_t UpperBound = Info.UpperBound;
-- 
cgit v1.2.3


From 94865edfa85a61dd4ad985d2fb86990a1bba357b Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <95053726+gandhi56@users.noreply.github.com>
Date: Fri, 20 Jun 2025 18:03:54 -0400
Subject: [Reland][InstCombine] Iterative replacement in PtrReplacer (#144626)

This patch enhances the PtrReplacer as follows:
1. Users are now collected iteratively to be generous on the stack. In the case of PHIs with incoming values which have not yet been visited, they are pushed back into the stack for reconsideration.
2. Replace users of the pointer root in a reverse-postorder traversal, instead of a simpletraversal over the collected users. This reordering ensures that the uses of an instruction are replaced before replacing the instruction itself.
3. During the replacement of PHI, use the same incoming value if it does not have a replacement.

This patch specifically fixes the case when an incoming value of a PHI
is addrspacecasted.

This is a reland of https://github.com/llvm/llvm-project/pull/137215.
---
 .../InstCombine/InstCombineLoadStoreAlloca.cpp     | 165 ++++++++++++---------
 .../InstCombine/AMDGPU/ptr-replace-alloca.ll       |  79 ++++++++++
 2 files changed, 175 insertions(+), 69 deletions(-)
 create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index a9751ab03e20..9aec90120d8b 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -243,11 +243,10 @@ public:
   void replacePointer(Value *V);
 
 private:
-  bool collectUsersRecursive(Instruction &I);
   void replace(Instruction *I);
-  Value *getReplacement(Value *I);
+  Value *getReplacement(Value *V) const { return WorkMap.lookup(V); }
   bool isAvailable(Instruction *I) const {
-    return I == &Root || Worklist.contains(I);
+    return I == &Root || UsersToReplace.contains(I);
   }
 
   bool isEqualOrValidAddrSpaceCast(const Instruction *I,
@@ -259,8 +258,7 @@ private:
     return (FromAS == ToAS) || IC.isValidAddrSpaceCast(FromAS, ToAS);
   }
 
-  SmallPtrSet<Instruction *, 32> ValuesToRevisit;
-  SmallSetVector<Instruction *, 4> Worklist;
+  SmallSetVector<Instruction *, 32> UsersToReplace;
   MapVector<Value *, Value *> WorkMap;
   InstCombinerImpl &IC;
   Instruction &Root;
@@ -269,72 +267,79 @@ private:
 } // end anonymous namespace
 
 bool PointerReplacer::collectUsers() {
-  if (!collectUsersRecursive(Root))
-    return false;
-
-  // Ensure that all outstanding (indirect) users of I
-  // are inserted into the Worklist. Return false
-  // otherwise.
-  return llvm::set_is_subset(ValuesToRevisit, Worklist);
-}
+  SmallVector<Instruction *> Worklist;
+  SmallSetVector<Instruction *, 32> ValuesToRevisit;
+
+  auto PushUsersToWorklist = [&](Instruction *Inst) {
+    for (auto *U : Inst->users())
+      if (auto *I = dyn_cast<Instruction>(U))
+        if (!isAvailable(I) && !ValuesToRevisit.contains(I))
+          Worklist.emplace_back(I);
+  };
 
-bool PointerReplacer::collectUsersRecursive(Instruction &I) {
-  for (auto *U : I.users()) {
-    auto *Inst = cast<Instruction>(&*U);
+  PushUsersToWorklist(&Root);
+  while (!Worklist.empty()) {
+    Instruction *Inst = Worklist.pop_back_val();
     if (auto *Load = dyn_cast<LoadInst>(Inst)) {
       if (Load->isVolatile())
         return false;
-      Worklist.insert(Load);
+      UsersToReplace.insert(Load);
     } else if (auto *PHI = dyn_cast<PHINode>(Inst)) {
-      // All incoming values must be instructions for replacability
-      if (any_of(PHI->incoming_values(),
-                 [](Value *V) { return !isa<Instruction>(V); }))
-        return false;
-
-      // If at least one incoming value of the PHI is not in Worklist,
-      // store the PHI for revisiting and skip this iteration of the
-      // loop.
-      if (any_of(PHI->incoming_values(), [this](Value *V) {
-            return !isAvailable(cast<Instruction>(V));
+      /// TODO: Handle poison and null pointers for PHI and select.
+      // If all incoming values are available, mark this PHI as
+      // replacable and push it's users into the worklist.
+      bool IsReplacable = true;
+      if (all_of(PHI->incoming_values(), [&](Value *V) {
+            if (!isa<Instruction>(V))
+              return IsReplacable = false;
+            return isAvailable(cast<Instruction>(V));
           })) {
-        ValuesToRevisit.insert(Inst);
+        UsersToReplace.insert(PHI);
+        PushUsersToWorklist(PHI);
         continue;
       }
 
-      Worklist.insert(PHI);
-      if (!collectUsersRecursive(*PHI))
-        return false;
-    } else if (auto *SI = dyn_cast<SelectInst>(Inst)) {
-      if (!isa<Instruction>(SI->getTrueValue()) ||
-          !isa<Instruction>(SI->getFalseValue()))
+      // Either an incoming value is not an instruction or not all
+      // incoming values are available. If this PHI was already
+      // visited prior to this iteration, return false.
+      if (!IsReplacable || !ValuesToRevisit.insert(PHI))
         return false;
 
-      if (!isAvailable(cast<Instruction>(SI->getTrueValue())) ||
-          !isAvailable(cast<Instruction>(SI->getFalseValue()))) {
-        ValuesToRevisit.insert(Inst);
-        continue;
+      // Push PHI back into the stack, followed by unavailable
+      // incoming values.
+      Worklist.emplace_back(PHI);
+      for (unsigned Idx = 0; Idx < PHI->getNumIncomingValues(); ++Idx) {
+        auto *IncomingValue = cast<Instruction>(PHI->getIncomingValue(Idx));
+        if (UsersToReplace.contains(IncomingValue))
+          continue;
+        if (!ValuesToRevisit.insert(IncomingValue))
+          return false;
+        Worklist.emplace_back(IncomingValue);
       }
-      Worklist.insert(SI);
-      if (!collectUsersRecursive(*SI))
-        return false;
-    } else if (isa<GetElementPtrInst>(Inst)) {
-      Worklist.insert(Inst);
-      if (!collectUsersRecursive(*Inst))
+    } else if (auto *SI = dyn_cast<SelectInst>(Inst)) {
+      auto *TrueInst = dyn_cast<Instruction>(SI->getTrueValue());
+      auto *FalseInst = dyn_cast<Instruction>(SI->getFalseValue());
+      if (!TrueInst || !FalseInst)
         return false;
+
+      UsersToReplace.insert(SI);
+      PushUsersToWorklist(SI);
+    } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
+      UsersToReplace.insert(GEP);
+      PushUsersToWorklist(GEP);
     } else if (auto *MI = dyn_cast<MemTransferInst>(Inst)) {
       if (MI->isVolatile())
         return false;
-      Worklist.insert(Inst);
+      UsersToReplace.insert(Inst);
     } else if (isEqualOrValidAddrSpaceCast(Inst, FromAS)) {
-      Worklist.insert(Inst);
-      if (!collectUsersRecursive(*Inst))
-        return false;
+      UsersToReplace.insert(Inst);
+      PushUsersToWorklist(Inst);
     } else if (Inst->isLifetimeStartOrEnd()) {
       continue;
     } else {
       // TODO: For arbitrary uses with address space mismatches, should we check
       // if we can introduce a valid addrspacecast?
-      LLVM_DEBUG(dbgs() << "Cannot handle pointer user: " << *U << '\n');
+      LLVM_DEBUG(dbgs() << "Cannot handle pointer user: " << *Inst << '\n');
       return false;
     }
   }
@@ -342,7 +347,39 @@ bool PointerReplacer::collectUsersRecursive(Instruction &I) {
   return true;
 }
 
-Value *PointerReplacer::getReplacement(Value *V) { return WorkMap.lookup(V); }
+void PointerReplacer::replacePointer(Value *V) {
+  assert(cast<PointerType>(Root.getType()) != cast<PointerType>(V->getType()) &&
+         "Invalid usage");
+  WorkMap[&Root] = V;
+  SmallVector<Instruction *> Worklist;
+  SetVector<Instruction *> PostOrderWorklist;
+  SmallPtrSet<Instruction *, 32> Visited;
+
+  // Perform a postorder traversal of the users of Root.
+  Worklist.push_back(&Root);
+  while (!Worklist.empty()) {
+    Instruction *I = Worklist.back();
+
+    // If I has not been processed before, push each of its
+    // replacable users into the worklist.
+    if (Visited.insert(I).second) {
+      for (auto *U : I->users()) {
+        auto *UserInst = cast<Instruction>(U);
+        if (UsersToReplace.contains(UserInst))
+          Worklist.push_back(UserInst);
+      }
+      // Otherwise, users of I have already been pushed into
+      // the PostOrderWorklist. Push I as well.
+    } else {
+      PostOrderWorklist.insert(I);
+      Worklist.pop_back();
+    }
+  }
+
+  // Replace pointers in reverse-postorder.
+  for (Instruction *I : reverse(PostOrderWorklist))
+    replace(I);
+}
 
 void PointerReplacer::replace(Instruction *I) {
   if (getReplacement(I))
@@ -364,13 +401,15 @@ void PointerReplacer::replace(Instruction *I) {
     // replacement (new value).
     WorkMap[NewI] = NewI;
   } else if (auto *PHI = dyn_cast<PHINode>(I)) {
-    Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType();
-    auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(),
-                                   PHI->getName(), PHI->getIterator());
-    for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I)
-      NewPHI->addIncoming(getReplacement(PHI->getIncomingValue(I)),
-                          PHI->getIncomingBlock(I));
-    WorkMap[PHI] = NewPHI;
+    // Create a new PHI by replacing any incoming value that is a user of the
+    // root pointer and has a replacement.
+    Value *V = WorkMap.lookup(PHI->getIncomingValue(0));
+    PHI->mutateType(V ? V->getType() : PHI->getIncomingValue(0)->getType());
+    for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I) {
+      Value *V = WorkMap.lookup(PHI->getIncomingValue(I));
+      PHI->setIncomingValue(I, V ? V : PHI->getIncomingValue(I));
+    }
+    WorkMap[PHI] = PHI;
   } else if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
     auto *V = getReplacement(GEP->getPointerOperand());
     assert(V && "Operand not replaced");
@@ -434,18 +473,6 @@ void PointerReplacer::replace(Instruction *I) {
   }
 }
 
-void PointerReplacer::replacePointer(Value *V) {
-#ifndef NDEBUG
-  auto *PT = cast<PointerType>(Root.getType());
-  auto *NT = cast<PointerType>(V->getType());
-  assert(PT != NT && "Invalid usage");
-#endif
-  WorkMap[&Root] = V;
-
-  for (Instruction *Workitem : Worklist)
-    replace(Workitem);
-}
-
 Instruction *InstCombinerImpl::visitAllocaInst(AllocaInst &AI) {
   if (auto *I = simplifyAllocaArraySize(*this, AI, DT))
     return I;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll
new file mode 100644
index 000000000000..538cc19f9722
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll
@@ -0,0 +1,79 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=instcombine -S < %s | FileCheck %s
+
+%struct.type = type { [256 x <2 x i64>] }
+@g1 = external hidden addrspace(3) global %struct.type, align 16
+
+; This test requires the PtrReplacer to replace users in an RPO traversal.
+; Furthermore, %ptr.else need not to be replaced so it must be retained in
+; %ptr.sink.
+define <2 x i64> @func(ptr addrspace(4) byref(%struct.type) align 16 %0, i1 %cmp.0) {
+; CHECK-LABEL: define <2 x i64> @func(
+; CHECK-SAME: ptr addrspace(4) byref([[STRUCT_TYPE:%.*]]) align 16 [[TMP0:%.*]], i1 [[CMP_0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    br i1 [[CMP_0]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
+; CHECK:       [[IF_THEN]]:
+; CHECK-NEXT:    [[VAL_THEN:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr
+; CHECK-NEXT:    br label %[[SINK:.*]]
+; CHECK:       [[IF_ELSE]]:
+; CHECK-NEXT:    [[PTR_ELSE:%.*]] = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
+; CHECK-NEXT:    br label %[[SINK]]
+; CHECK:       [[SINK]]:
+; CHECK-NEXT:    [[PTR_SINK:%.*]] = phi ptr [ [[PTR_ELSE]], %[[IF_ELSE]] ], [ [[VAL_THEN]], %[[IF_THEN]] ]
+; CHECK-NEXT:    [[VAL_SINK:%.*]] = load <2 x i64>, ptr [[PTR_SINK]], align 16
+; CHECK-NEXT:    ret <2 x i64> [[VAL_SINK]]
+;
+entry:
+  %coerce = alloca %struct.type, align 16, addrspace(5)
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 16 %coerce, ptr addrspace(4) align 16 %0, i64 4096, i1 false)
+  br i1 %cmp.0, label %if.then, label %if.else
+
+if.then:                                    ; preds = %entry
+  %ptr.then = getelementptr inbounds i8, ptr addrspace(5) %coerce, i64 0
+  %val.then = addrspacecast ptr addrspace(5) %ptr.then to ptr
+  br label %sink
+
+if.else:                                      ; preds = %entry
+  %ptr.else = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
+  %val.else = getelementptr inbounds nuw i8, ptr %ptr.else, i64 0
+  br label %sink
+
+sink:
+  %ptr.sink = phi ptr [ %val.else, %if.else ], [ %val.then, %if.then ]
+  %val.sink = load <2 x i64>, ptr %ptr.sink, align 16
+  ret <2 x i64> %val.sink
+}
+
+define <2 x i64> @func_phi_loop(ptr addrspace(4) byref(%struct.type) align 16 %0, i1 %cmp.0) {
+; CHECK-LABEL: define <2 x i64> @func_phi_loop(
+; CHECK-SAME: ptr addrspace(4) byref([[STRUCT_TYPE:%.*]]) align 16 [[TMP0:%.*]], i1 [[CMP_0:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*]]:
+; CHECK-NEXT:    [[VAL_0:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr
+; CHECK-NEXT:    br label %[[LOOP:.*]]
+; CHECK:       [[LOOP]]:
+; CHECK-NEXT:    [[PTR_PHI_R:%.*]] = phi ptr [ [[PTR_1:%.*]], %[[LOOP]] ], [ [[VAL_0]], %[[ENTRY]] ]
+; CHECK-NEXT:    [[PTR_1]] = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
+; CHECK-NEXT:    br i1 [[CMP_0]], label %[[LOOP]], label %[[SINK:.*]]
+; CHECK:       [[SINK]]:
+; CHECK-NEXT:    [[VAL_SINK:%.*]] = load <2 x i64>, ptr [[PTR_PHI_R]], align 16
+; CHECK-NEXT:    ret <2 x i64> [[VAL_SINK]]
+;
+entry:
+  %coerce = alloca %struct.type, align 16, addrspace(5)
+  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 16 %coerce, ptr addrspace(4) align 16 %0, i64 4096, i1 false)
+  %ptr.0 = getelementptr inbounds i8, ptr addrspace(5) %coerce, i64 0
+  %val.0 = addrspacecast ptr addrspace(5) %ptr.0 to ptr
+  br label %loop
+
+loop:
+  %ptr.phi = phi ptr [ %val.1, %loop ], [ %val.0, %entry ]
+  %ptr.1 = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
+  %val.1 = getelementptr inbounds nuw i8, ptr %ptr.1, i64 0
+  br i1 %cmp.0, label %loop, label %sink
+
+sink:
+  %val.sink = load <2 x i64>, ptr %ptr.phi, align 16
+  ret <2 x i64> %val.sink
+}
+
+declare void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noalias writeonly captures(none), ptr addrspace(4) noalias readonly captures(none), i64, i1 immarg) #0
-- 
cgit v1.2.3


From b7be8786af42d131974ec9cfc3ba79b264511b7b Mon Sep 17 00:00:00 2001
From: Aiden Grossman <aidengrossman@google.com>
Date: Fri, 20 Jun 2025 15:07:00 -0700
Subject: Reapply "[CI] Migrate to runtimes build" (#143612)

This reverts commit 6f62979a5a5bcf70d65f23e0991a274e6df5955b.

The reapplies commit 80ea5f46df3e365a0a2112889bb91732167b6214.

That commit was reverted because it was causing compiler-rt test
failures due to tysan not having its dependencies set up properly within
CMake. That situation has since been rectified in
3cef099ceddccefca8e11268624397cde9e04af6.

Reviewers: lnihlen, rnk, gburgessiv, cmtice

Reviewed By: rnk, cmtice

Pull Request: https://github.com/llvm/llvm-project/pull/144033
---
 .ci/compute_projects.py         | 115 ++++++++++++++++++++++++++--------------
 .ci/compute_projects_test.py    |  55 +++++++++++++++++--
 .ci/monolithic-linux.sh         |  13 +++--
 .github/workflows/premerge.yaml |   3 +-
 4 files changed, 137 insertions(+), 49 deletions(-)

diff --git a/.ci/compute_projects.py b/.ci/compute_projects.py
index 40dd0507a9ea..e61b8dc5021f 100644
--- a/.ci/compute_projects.py
+++ b/.ci/compute_projects.py
@@ -49,8 +49,7 @@ DEPENDENTS_TO_TEST = {
     },
     "lld": {"bolt", "cross-project-tests"},
     # TODO(issues/132795): LLDB should be enabled on clang changes.
-    "clang": {"clang-tools-extra", "compiler-rt", "cross-project-tests"},
-    "clang-tools-extra": {"libc"},
+    "clang": {"clang-tools-extra", "cross-project-tests"},
     "mlir": {"flang"},
     # Test everything if ci scripts are changed.
     # FIXME: Figure out what is missing and add here.
@@ -64,7 +63,15 @@ DEPENDENT_RUNTIMES_TO_BUILD = {"lldb": {"libcxx", "libcxxabi", "libunwind"}}
 
 # This mapping describes runtimes that should be tested when the key project is
 # touched.
-DEPENDENT_RUNTIMES_TO_TEST = {"clang": {"libcxx", "libcxxabi", "libunwind"}}
+DEPENDENT_RUNTIMES_TO_TEST = {
+    "clang": {"compiler-rt"},
+    "clang-tools-extra": {"libc"},
+}
+DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG = {
+    "llvm": {"libcxx", "libcxxabi", "libunwind"},
+    "clang": {"libcxx", "libcxxabi", "libunwind"},
+    ".ci": {"libcxx", "libcxxabi", "libunwind"},
+}
 
 EXCLUDE_LINUX = {
     "cross-project-tests",  # TODO(issues/132796): Tests are failing.
@@ -93,9 +100,6 @@ EXCLUDE_MAC = {
     "cross-project-tests",
     "flang",
     "libc",
-    "libcxx",
-    "libcxxabi",
-    "libunwind",
     "lldb",
     "openmp",
     "polly",
@@ -122,10 +126,10 @@ PROJECT_CHECK_TARGETS = {
     "polly": "check-polly",
 }
 
-RUNTIMES = {"libcxx", "libcxxabi", "libunwind"}
+RUNTIMES = {"libcxx", "libcxxabi", "libunwind", "compiler-rt", "libc"}
 
 
-def _add_dependencies(projects: Set[str]) -> Set[str]:
+def _add_dependencies(projects: Set[str], runtimes: Set[str]) -> Set[str]:
     projects_with_dependents = set(projects)
     current_projects_count = 0
     while current_projects_count != len(projects_with_dependents):
@@ -134,9 +138,25 @@ def _add_dependencies(projects: Set[str]) -> Set[str]:
             if project not in PROJECT_DEPENDENCIES:
                 continue
             projects_with_dependents.update(PROJECT_DEPENDENCIES[project])
+    for runtime in runtimes:
+        if runtime not in PROJECT_DEPENDENCIES:
+            continue
+        projects_with_dependents.update(PROJECT_DEPENDENCIES[runtime])
     return projects_with_dependents
 
 
+def _exclude_projects(current_projects: Set[str], platform: str) -> Set[str]:
+    if platform == "Linux":
+        to_exclude = EXCLUDE_LINUX
+    elif platform == "Windows":
+        to_exclude = EXCLUDE_WINDOWS
+    elif platform == "Darwin":
+        to_exclude = EXCLUDE_MAC
+    else:
+        raise ValueError(f"Unexpected platform: {platform}")
+    return current_projects.difference(to_exclude)
+
+
 def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     projects_to_test = set()
     for modified_project in modified_projects:
@@ -154,25 +174,14 @@ def _compute_projects_to_test(modified_projects: Set[str], platform: str) -> Set
             ):
                 continue
             projects_to_test.add(dependent_project)
-    if platform == "Linux":
-        for to_exclude in EXCLUDE_LINUX:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    elif platform == "Windows":
-        for to_exclude in EXCLUDE_WINDOWS:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    elif platform == "Darwin":
-        for to_exclude in EXCLUDE_MAC:
-            if to_exclude in projects_to_test:
-                projects_to_test.remove(to_exclude)
-    else:
-        raise ValueError("Unexpected platform.")
+    projects_to_test = _exclude_projects(projects_to_test, platform)
     return projects_to_test
 
 
-def _compute_projects_to_build(projects_to_test: Set[str]) -> Set[str]:
-    return _add_dependencies(projects_to_test)
+def _compute_projects_to_build(
+    projects_to_test: Set[str], runtimes: Set[str]
+) -> Set[str]:
+    return _add_dependencies(projects_to_test, runtimes)
 
 
 def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
@@ -184,24 +193,36 @@ def _compute_project_check_targets(projects_to_test: Set[str]) -> Set[str]:
     return check_targets
 
 
-def _compute_runtimes_to_test(projects_to_test: Set[str]) -> Set[str]:
+def _compute_runtimes_to_test(modified_projects: Set[str], platform: str) -> Set[str]:
     runtimes_to_test = set()
-    for project_to_test in projects_to_test:
-        if project_to_test in DEPENDENT_RUNTIMES_TO_TEST:
-            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[project_to_test])
-        if project_to_test in DEPENDENT_RUNTIMES_TO_BUILD:
-            runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_BUILD[project_to_test])
-    return runtimes_to_test
+    for modified_project in modified_projects:
+        if modified_project not in DEPENDENT_RUNTIMES_TO_TEST:
+            continue
+        runtimes_to_test.update(DEPENDENT_RUNTIMES_TO_TEST[modified_project])
+    return _exclude_projects(runtimes_to_test, platform)
 
 
-def _compute_runtime_check_targets(projects_to_test: Set[str]) -> Set[str]:
-    check_targets = set()
-    for project_to_test in projects_to_test:
-        if project_to_test not in DEPENDENT_RUNTIMES_TO_TEST:
+def _compute_runtimes_to_test_needs_reconfig(
+    modified_projects: Set[str], platform: str
+) -> Set[str]:
+    runtimes_to_test = set()
+    for modified_project in modified_projects:
+        if modified_project not in DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG:
             continue
-        for runtime_to_test in DEPENDENT_RUNTIMES_TO_TEST[project_to_test]:
-            check_targets.add(PROJECT_CHECK_TARGETS[runtime_to_test])
-    return check_targets
+        runtimes_to_test.update(
+            DEPENDENT_RUNTIMES_TO_TEST_NEEDS_RECONFIG[modified_project]
+        )
+    return _exclude_projects(runtimes_to_test, platform)
+
+
+def _compute_runtimes_to_build(
+    runtimes_to_test: Set[str], modified_projects: Set[str], platform: str
+) -> Set[str]:
+    runtimes_to_build = set(runtimes_to_test)
+    for modified_project in modified_projects:
+        if modified_project in DEPENDENT_RUNTIMES_TO_BUILD:
+            runtimes_to_build.update(DEPENDENT_RUNTIMES_TO_BUILD[modified_project])
+    return _exclude_projects(runtimes_to_build, platform)
 
 
 def _get_modified_projects(modified_files: list[str]) -> Set[str]:
@@ -225,10 +246,19 @@ def _get_modified_projects(modified_files: list[str]) -> Set[str]:
 def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
     modified_projects = _get_modified_projects(modified_files)
     projects_to_test = _compute_projects_to_test(modified_projects, platform)
-    projects_to_build = _compute_projects_to_build(projects_to_test)
+    runtimes_to_test = _compute_runtimes_to_test(modified_projects, platform)
+    runtimes_to_test_needs_reconfig = _compute_runtimes_to_test_needs_reconfig(
+        modified_projects, platform
+    )
+    runtimes_to_build = _compute_runtimes_to_build(
+        runtimes_to_test | runtimes_to_test_needs_reconfig, modified_projects, platform
+    )
+    projects_to_build = _compute_projects_to_build(projects_to_test, runtimes_to_build)
     projects_check_targets = _compute_project_check_targets(projects_to_test)
-    runtimes_to_build = _compute_runtimes_to_test(projects_to_test)
-    runtimes_check_targets = _compute_runtime_check_targets(projects_to_test)
+    runtimes_check_targets = _compute_project_check_targets(runtimes_to_test)
+    runtimes_check_targets_needs_reconfig = _compute_project_check_targets(
+        runtimes_to_test_needs_reconfig
+    )
     # We use a semicolon to separate the projects/runtimes as they get passed
     # to the CMake invocation and thus we need to use the CMake list separator
     # (;). We use spaces to separate the check targets as they end up getting
@@ -238,6 +268,9 @@ def get_env_variables(modified_files: list[str], platform: str) -> Set[str]:
         "project_check_targets": " ".join(sorted(projects_check_targets)),
         "runtimes_to_build": ";".join(sorted(runtimes_to_build)),
         "runtimes_check_targets": " ".join(sorted(runtimes_check_targets)),
+        "runtimes_check_targets_needs_reconfig": " ".join(
+            sorted(runtimes_check_targets_needs_reconfig)
+        ),
     }
 
 
diff --git a/.ci/compute_projects_test.py b/.ci/compute_projects_test.py
index ae376ea6a43c..6bc2e34a1cbe 100644
--- a/.ci/compute_projects_test.py
+++ b/.ci/compute_projects_test.py
@@ -26,6 +26,10 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -46,6 +50,10 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -66,6 +74,10 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -75,17 +87,21 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(
             env_variables["projects_to_build"],
-            "clang;clang-tools-extra;compiler-rt;lld;llvm",
+            "clang;clang-tools-extra;lld;llvm",
         )
         self.assertEqual(
             env_variables["project_check_targets"],
-            "check-clang check-clang-tools check-compiler-rt",
+            "check-clang check-clang-tools",
         )
         self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+            env_variables["runtimes_to_build"], "compiler-rt;libcxx;libcxxabi;libunwind"
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "check-compiler-rt",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -104,6 +120,10 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -115,6 +135,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "check-bolt")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_lldb(self):
         env_variables = compute_projects.get_env_variables(
@@ -124,6 +145,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "check-lldb")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_mlir(self):
         env_variables = compute_projects.get_env_variables(
@@ -135,6 +157,7 @@ class TestComputeProjects(unittest.TestCase):
         )
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_flang(self):
         env_variables = compute_projects.get_env_variables(
@@ -144,6 +167,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "check-flang")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_invalid_subproject(self):
         env_variables = compute_projects.get_env_variables(
@@ -153,6 +177,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_top_level_file(self):
         env_variables = compute_projects.get_env_variables(["README.md"], "Linux")
@@ -160,6 +185,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_runtiems_in_projects(self):
         env_variables = compute_projects.get_env_variables(
@@ -169,6 +195,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_docs(self):
         env_variables = compute_projects.get_env_variables(
@@ -178,6 +205,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_exclude_gn(self):
         env_variables = compute_projects.get_env_variables(
@@ -187,6 +215,7 @@ class TestComputeProjects(unittest.TestCase):
         self.assertEqual(env_variables["project_check_targets"], "")
         self.assertEqual(env_variables["runtimes_to_build"], "")
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
     def test_ci(self):
         env_variables = compute_projects.get_env_variables(
@@ -198,10 +227,15 @@ class TestComputeProjects(unittest.TestCase):
             "check-clang check-lld check-lldb check-llvm",
         )
         self.assertEqual(
-            env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
+            env_variables["runtimes_to_build"],
+            "libcxx;libcxxabi;libunwind",
         )
         self.assertEqual(
             env_variables["runtimes_check_targets"],
+            "",
+        )
+        self.assertEqual(
+            env_variables["runtimes_check_targets_needs_reconfig"],
             "check-cxx check-cxxabi check-unwind",
         )
 
@@ -215,6 +249,19 @@ class TestComputeProjects(unittest.TestCase):
             env_variables["runtimes_to_build"], "libcxx;libcxxabi;libunwind"
         )
         self.assertEqual(env_variables["runtimes_check_targets"], "")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
+
+    def test_clang_tools_extra(self):
+        env_variables = compute_projects.get_env_variables(
+            ["clang-tools-extra/CMakeLists.txt"], "Linux"
+        )
+        self.assertEqual(
+            env_variables["projects_to_build"], "clang;clang-tools-extra;lld;llvm"
+        )
+        self.assertEqual(env_variables["project_check_targets"], "check-clang-tools")
+        self.assertEqual(env_variables["runtimes_to_build"], "libc")
+        self.assertEqual(env_variables["runtimes_check_targets"], "check-libc")
+        self.assertEqual(env_variables["runtimes_check_targets_needs_reconfig"], "")
 
 
 if __name__ == "__main__":
diff --git a/.ci/monolithic-linux.sh b/.ci/monolithic-linux.sh
index 7503ea4e6a99..c350a5867914 100755
--- a/.ci/monolithic-linux.sh
+++ b/.ci/monolithic-linux.sh
@@ -57,6 +57,7 @@ projects="${1}"
 targets="${2}"
 runtimes="${3}"
 runtime_targets="${4}"
+runtime_targets_needs_reconfig="${5}"
 
 lit_args="-v --xunit-xml-output ${BUILD_DIR}/test-results.xml --use-unique-output-file-name --timeout=1200 --time-tests"
 
@@ -93,9 +94,15 @@ echo "--- ninja"
 # Targets are not escaped as they are passed as separate arguments.
 ninja -C "${BUILD_DIR}" -k 0 ${targets}
 
+if [[ "${runtime_targets}" != "" ]]; then
+  echo "--- ninja runtimes"
+
+  ninja -C "${BUILD_DIR}" ${runtime_targets}
+fi
+
 # Compiling runtimes with just-built Clang and running their tests
 # as an additional testing for Clang.
-if [[ "${runtimes_targets}" != "" ]]; then
+if [[ "${runtime_targets_needs_reconfig}" != "" ]]; then
   echo "--- cmake runtimes C++26"
 
   cmake \
@@ -105,7 +112,7 @@ if [[ "${runtimes_targets}" != "" ]]; then
 
   echo "--- ninja runtimes C++26"
 
-  ninja -C "${BUILD_DIR}" ${runtime_targets}
+  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
 
   echo "--- cmake runtimes clang modules"
 
@@ -116,5 +123,5 @@ if [[ "${runtimes_targets}" != "" ]]; then
 
   echo "--- ninja runtimes clang modules"
 
-  ninja -C "${BUILD_DIR}" ${runtime_targets}
+  ninja -C "${BUILD_DIR}" ${runtime_targets_needs_reconfig}
 fi
diff --git a/.github/workflows/premerge.yaml b/.github/workflows/premerge.yaml
index 709b6d03d94c..4435a3e90576 100644
--- a/.github/workflows/premerge.yaml
+++ b/.github/workflows/premerge.yaml
@@ -56,11 +56,12 @@ jobs:
           echo "Running project checks targets: ${project_check_targets}"
           echo "Building runtimes: ${runtimes_to_build}"
           echo "Running runtimes checks targets: ${runtimes_check_targets}"
+          echo "Running runtimes checks requiring reconfiguring targets: ${runtimes_check_targets_needs_reconfig}"
 
           export CC=/opt/llvm/bin/clang
           export CXX=/opt/llvm/bin/clang++
 
-          ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
+          ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}" "${runtimes_check_targets_needs_reconfig}"
       - name: Upload Artifacts
         uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
         with:
-- 
cgit v1.2.3


From 1db9afb102acbb447ee374851d11fcdbea7fe5ec Mon Sep 17 00:00:00 2001
From: Med Ismail Bennani <ismail@bennani.ma>
Date: Fri, 20 Jun 2025 15:26:14 -0700
Subject: [lldb/crashlog] Make registers always available & fix x29/x30 parsing
 (#145104)

This patch addresses 2 issues:
1. It makes registers available on non-crashed threads all the time
2. It fixes arm64 registers parsing for registers that don't use the `x`
prefix (`fp` -> `x29` / `lr` -> `x30`)

---------

Signed-off-by: Med Ismail Bennani <ismail@bennani.ma>
---
 lldb/examples/python/crashlog.py                   |  8 +--
 lldb/examples/python/crashlog_scripted_process.py  | 12 ++--
 .../interactive_crashlog_arm64_register.test       | 80 ++++++++++++++++++++++
 3 files changed, 91 insertions(+), 9 deletions(-)
 create mode 100644 lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_arm64_register.test

diff --git a/lldb/examples/python/crashlog.py b/lldb/examples/python/crashlog.py
index 5f07cda2892a..bb20f3a25c1c 100755
--- a/lldb/examples/python/crashlog.py
+++ b/lldb/examples/python/crashlog.py
@@ -777,10 +777,10 @@ class JSONCrashLogParser(CrashLogParser):
             if json_thread.get("triggered", False):
                 self.crashlog.crashed_thread_idx = idx
                 thread.crashed = True
-                if "threadState" in json_thread:
-                    thread.registers = self.parse_thread_registers(
-                        json_thread["threadState"]
-                    )
+            if "threadState" in json_thread:
+                thread.registers = self.parse_thread_registers(
+                    json_thread["threadState"]
+                )
             if "queue" in json_thread:
                 thread.queue = json_thread.get("queue")
             self.parse_frames(thread, json_thread.get("frames", []))
diff --git a/lldb/examples/python/crashlog_scripted_process.py b/lldb/examples/python/crashlog_scripted_process.py
index be0ed49d3590..f54a8df0479e 100644
--- a/lldb/examples/python/crashlog_scripted_process.py
+++ b/lldb/examples/python/crashlog_scripted_process.py
@@ -123,11 +123,6 @@ class CrashLogScriptedProcess(ScriptedProcess):
 
 class CrashLogScriptedThread(ScriptedThread):
     def create_register_ctx(self):
-        if not self.has_crashed:
-            return dict.fromkeys(
-                [*map(lambda reg: reg["name"], self.register_info["registers"])], 0
-            )
-
         if not self.backing_thread or not len(self.backing_thread.registers):
             return dict.fromkeys(
                 [*map(lambda reg: reg["name"], self.register_info["registers"])], 0
@@ -135,8 +130,15 @@ class CrashLogScriptedThread(ScriptedThread):
 
         for reg in self.register_info["registers"]:
             reg_name = reg["name"]
+            reg_alt_name = None
+            if "alt-name" in reg:
+                reg_alt_name = reg["alt-name"]
             if reg_name in self.backing_thread.registers:
                 self.register_ctx[reg_name] = self.backing_thread.registers[reg_name]
+            elif reg_alt_name and reg_alt_name in self.backing_thread.registers:
+                self.register_ctx[reg_name] = self.backing_thread.registers[
+                    reg_alt_name
+                ]
             else:
                 self.register_ctx[reg_name] = 0
 
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_arm64_register.test b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_arm64_register.test
new file mode 100644
index 000000000000..3f572c3300c0
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Crashlog/interactive_crashlog_arm64_register.test
@@ -0,0 +1,80 @@
+# REQUIRES: python, native && system-darwin
+
+# RUN: mkdir -p %t.dir
+# RUN: yaml2obj %S/Inputs/interactive_crashlog/multithread-test.yaml > %t.dir/multithread-test
+# RUN: %lldb -o 'command script import lldb.macosx.crashlog' \
+# RUN: -o 'crashlog -a -t %t.dir/multithread-test %S/Inputs/interactive_crashlog/multithread-test.ips' \
+# RUN: -o "thread list" -o "bt all" -o "register read" 2>&1 | FileCheck %s
+
+# CHECK: "crashlog" {{.*}} commands have been installed, use the "--help" options on these commands
+
+# CHECK: (lldb) process status
+# CHECK-NEXT: Process 22511 stopped
+# CHECK-NEXT: * thread #3, stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
+# CHECK-NEXT:     frame #0: 0x0000000100ec58f4 multithread-test`bar
+
+# CHECK: (lldb) thread backtrace
+# CHECK-NEXT: * thread #3, stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
+# CHECK-NEXT:   * frame #0: 0x0000000100ec58f4 multithread-test`bar{{.*}} [artificial]
+# CHECK-NEXT:     frame #1: 0x0000000100ec591b multithread-test`foo{{.*}} [artificial]
+# CHECK-NEXT:     frame #2: 0x0000000100ec5a87 multithread-test`compute_pow{{.*}} [artificial]
+
+# CHECK: (lldb) thread list
+# CHECK-NEXT: Process 22511 stopped
+# CHECK-NEXT:   thread #1: tid = 0x23c7fe, 0x000000019cc40b84{{.*}}, queue = 'com.apple.main-thread'
+# CHECK-NEXT:   thread #2: tid = 0x23c800, 0x000000019cc42c9c{{.*}}
+# CHECK-NEXT: * thread #3: tid = 0x23c801, 0x0000000100ec58f4 multithread-test`bar{{.*}}, stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
+
+# CHECK: (lldb) bt all
+# CHECK:  thread #1, queue = 'com.apple.main-thread'
+# CHECK:    frame #{{[0-9]+}}: 0x000000019cc40b84{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x0000000100ec5b3b multithread-test`main{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x00000002230f8da7{{.*}} [artificial]
+# CHECK-NEXT:  thread #2
+# CHECK-NEXT:    frame #0: 0x000000019cc42c9c{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x0000000100ec5957 multithread-test`call_and_wait{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x000000019cc7e06b{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x000000019cc78e2b{{.*}} [artificial]
+# CHECK-NEXT:* thread #3, stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
+# CHECK-NEXT:  * frame #0: 0x0000000100ec58f4 multithread-test`bar{{.*}} [artificial]
+# CHECK-NEXT:    frame #1: 0x0000000100ec591b multithread-test`foo{{.*}} [artificial]
+# CHECK-NEXT:    frame #2: 0x0000000100ec5a87 multithread-test`compute_pow{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x000000019cc7e06b{{.*}} [artificial]
+# CHECK:    frame #{{[0-9]+}}: 0x000000019cc78e2b{{.*}} [artificial]
+
+# CHECK: (lldb) register read
+# CHECK: General Purpose Registers:
+# CHECK:         x0 = 0x000000000000002a
+# CHECK:         x1 = 0x0000600001d291b0
+# CHECK:         x2 = 0x000000019cbbf000
+# CHECK:         x3 = 0x0000000000000000
+# CHECK:         x4 = 0x00000000000030a0
+# CHECK:         x5 = 0x00000000190008ff
+# CHECK:         x6 = 0x0000000000000000
+# CHECK:         x7 = 0x0000000000000000
+# CHECK:         x8 = 0x0000000000000001
+# CHECK:         x9 = 0x0000000000000000
+# CHECK:        x10 = 0xfffffffe634277cf
+# CHECK:        x11 = 0x0000010000000102
+# CHECK:        x12 = 0x0000010000000102
+# CHECK:        x13 = 0x0000010000000100
+# CHECK:        x14 = 0x0000010000000000
+# CHECK:        x15 = 0x0000000000000001
+# CHECK:        x16 = 0x000000019cc78ea8
+# CHECK:        x17 = 0x00000001fd0a7698
+# CHECK:        x18 = 0x0000000000000000
+# CHECK:        x19 = 0x000000016f04f000
+# CHECK:        x20 = 0x0000000000000000
+# CHECK:        x21 = 0x0000000000000000
+# CHECK:        x22 = 0x0000000000000000
+# CHECK:        x23 = 0x0000000000000000
+# CHECK:        x24 = 0x0000000000000000
+# CHECK:        x25 = 0x0000000000000000
+# CHECK:        x26 = 0x0000000000000000
+# CHECK:        x27 = 0x0000000000000000
+# CHECK:        x28 = 0x0000000000000000
+# CHECK:        x29 = 0x000000016f04ef00
+# CHECK:        x30 = 0x0000000100ec591c
+# CHECK:         sp = 0x000000016f04eee0
+# CHECK:         pc = 0x0000000100ec58f4
+# CHECK:       cpsr = 0x80001000
-- 
cgit v1.2.3


From 1753aba0342bd286d65d1c8465b9fa2659424766 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Fri, 20 Jun 2025 23:38:30 +0100
Subject: [X86] combineINSERT_SUBVECTOR - directly fold to
 X86ISD::SUBV_BROADCAST_LOAD to prevent vector split infinite loop (#145077)

This reverts #140919 / f1d03dedfbe87119cfcafb07e0e0f90ec291cb97 - which
could result in another fold trying to split the concatenation apart
again before it was folded to a SUBV_BROADCAST_LOAD
---
 llvm/lib/Target/X86/X86ISelLowering.cpp            |  10 +-
 llvm/test/CodeGen/X86/oddshuffles.ll               |  47 ++
 .../X86/vector-interleaved-store-i8-stride-7.ll    | 556 ++++++++++-----------
 3 files changed, 332 insertions(+), 281 deletions(-)

diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 35d7b8084405..33083c0eba69 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59465,8 +59465,7 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
   }
 
   // If we're splatting the lower half subvector of a full vector load into the
-  // upper half, just splat the subvector directly, potentially creating a
-  // subvector broadcast.
+  // upper half, attempt to create a subvector broadcast.
   if ((int)IdxVal == (VecNumElts / 2) &&
       Vec.getValueSizeInBits() == (2 * SubVec.getValueSizeInBits())) {
     auto *VecLd = dyn_cast<LoadSDNode>(Vec);
@@ -59474,7 +59473,12 @@ static SDValue combineINSERT_SUBVECTOR(SDNode *N, SelectionDAG &DAG,
     if (VecLd && SubLd &&
         DAG.areNonVolatileConsecutiveLoads(
             SubLd, VecLd, SubVec.getValueSizeInBits() / 8, 0)) {
-      return concatSubVectors(SubVec, SubVec, DAG, dl);
+      SDValue BcastLd = getBROADCAST_LOAD(X86ISD::SUBV_BROADCAST_LOAD, dl, OpVT,
+                                          SubVecVT, SubLd, 0, DAG);
+      SDValue NewSubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT,
+                                      BcastLd, DAG.getVectorIdxConstant(0, dl));
+      DCI.CombineTo(SubLd, NewSubVec, BcastLd.getValue(1));
+      return BcastLd;
     }
   }
 
diff --git a/llvm/test/CodeGen/X86/oddshuffles.ll b/llvm/test/CodeGen/X86/oddshuffles.ll
index 6b9a86343ea1..4b0f75df83a7 100644
--- a/llvm/test/CodeGen/X86/oddshuffles.ll
+++ b/llvm/test/CodeGen/X86/oddshuffles.ll
@@ -2508,3 +2508,50 @@ define void @D107009(ptr %input, ptr %output) {
   store <64 x i32> %i7, ptr %output, align 16
   ret void
 }
+
+; Ensure concatenation of repeated subvector loads before vector can be split apart.
+define void @split_v2i64_subvector_broadcast(ptr readonly align 8 captures(none) dereferenceable(64) %arg) {
+; SSE-LABEL: split_v2i64_subvector_broadcast:
+; SSE:       # %bb.0:
+; SSE-NEXT:    movups 8(%rdi), %xmm0
+; SSE-NEXT:    movups 40(%rdi), %xmm1
+; SSE-NEXT:    movaps %xmm0, %xmm2
+; SSE-NEXT:    movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSE-NEXT:    unpckhpd {{.*#+}} xmm0 = xmm0[1],xmm1[1]
+; SSE-NEXT:    movups %xmm0, (%rax)
+; SSE-NEXT:    movups %xmm2, (%rax)
+; SSE-NEXT:    retq
+;
+; AVX1-LABEL: split_v2i64_subvector_broadcast:
+; AVX1:       # %bb.0:
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; AVX1-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; AVX1-NEXT:    vmovupd %ymm0, (%rax)
+; AVX1-NEXT:    vzeroupper
+; AVX1-NEXT:    retq
+;
+; AVX2-LABEL: split_v2i64_subvector_broadcast:
+; AVX2:       # %bb.0:
+; AVX2-NEXT:    vmovups 40(%rdi), %xmm0
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,0,2,1]
+; AVX2-NEXT:    vpermpd {{.*#+}} ymm1 = mem[0,1,1,3]
+; AVX2-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-NEXT:    vmovups %ymm0, (%rax)
+; AVX2-NEXT:    vzeroupper
+; AVX2-NEXT:    retq
+;
+; XOP-LABEL: split_v2i64_subvector_broadcast:
+; XOP:       # %bb.0:
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
+; XOP-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; XOP-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
+; XOP-NEXT:    vmovupd %ymm0, (%rax)
+; XOP-NEXT:    vzeroupper
+; XOP-NEXT:    retq
+  %gep = getelementptr inbounds nuw i8, ptr %arg, i64 8
+  %load = load <6 x i64>, ptr %gep, align 8
+  %shuffle = shufflevector <6 x i64> %load, <6 x i64> poison, <4 x i32> <i32 0, i32 4, i32 1, i32 5>
+  store <4 x i64> %shuffle, ptr poison, align 8
+  ret void
+}
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 25e489eef9d1..61bfee133d84 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -10105,14 +10105,14 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512BW-FCP:       # %bb.0:
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm15
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm1
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm3
-; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm4
+; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm4, %ymm1
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
 ; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
@@ -10122,105 +10122,106 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
-; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm5, %ymm6
-; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm6, %ymm3
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm5, %ymm5
-; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm6
+; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm5
+; AVX512BW-FCP-NEXT:    vpor %ymm4, %ymm5, %ymm4
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm6
+; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm6, %ymm1
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm5
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm4, %zmm4
 ; AVX512BW-FCP-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512BW-FCP-NEXT:    kmovq %r10, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm5 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm6
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm6
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm7
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm8
-; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm8, %ymm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm10
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm10
+; AVX512BW-FCP-NEXT:    vpor %ymm6, %ymm10, %ymm10
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm7, %ymm11
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm3, %ymm9, %ymm9
+; AVX512BW-FCP-NEXT:    vpor %ymm11, %ymm9, %ymm9
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,3,2,3]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm10, %zmm9
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm10
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm4, %ymm11
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10]
+; AVX512BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm11
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
 ; AVX512BW-FCP-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm3 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm9 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512BW-FCP-NEXT:    vpshufb %ymm9, %ymm11, %ymm3
-; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm18[8],xmm3[8],xmm18[9],xmm3[9],xmm18[10],xmm3[10],xmm18[11],xmm3[11],xmm18[12],xmm3[12],xmm18[13],xmm3[13],xmm18[14],xmm3[14],xmm18[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqa (%rdi), %ymm9
+; AVX512BW-FCP-NEXT:    vpshufb %ymm2, %ymm9, %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqa (%rsi), %ymm10
+; AVX512BW-FCP-NEXT:    vpshufb %ymm8, %ymm10, %ymm8
+; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm8, %ymm2
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm23
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm16
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm23[8],xmm18[8],xmm23[9],xmm18[9],xmm23[10],xmm18[10],xmm23[11],xmm18[11],xmm23[12],xmm18[12],xmm23[13],xmm18[13],xmm23[14],xmm18[14],xmm23[15],xmm18[15]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm9, %xmm9
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm9
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm8, %xmm8
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm8, %zmm8
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm2
 ; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
-; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm14
-; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm14, %ymm2
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm13
+; AVX512BW-FCP-NEXT:    vpor %ymm2, %ymm13, %ymm2
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm14, %xmm14
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
+; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm13, %xmm13
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
 ; AVX512BW-FCP-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm9 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm8 {%k2}
+; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
 ; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
-; AVX512BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
+; AVX512BW-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
 ; AVX512BW-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm1, %xmm1
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm8 {%k2}
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm18, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm23, %xmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm3, %xmm1
+; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm18, %xmm1
 ; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm18[0],xmm3[1],xmm18[1],xmm3[2],xmm18[2],xmm3[3],xmm18[3],xmm3[4],xmm18[4],xmm3[5],xmm18[5],xmm3[6],xmm18[6],xmm3[7],xmm18[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm18[0],xmm23[0],xmm18[1],xmm23[1],xmm18[2],xmm23[2],xmm18[3],xmm23[3],xmm18[4],xmm23[4],xmm18[5],xmm23[5],xmm18[6],xmm23[6],xmm18[7],xmm23[7]
 ; AVX512BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm1[0,1,0,1],zmm0[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm13, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm12, %xmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm27 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm12, %xmm1
+; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm11, %xmm1
 ; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3],xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm23 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
@@ -10228,48 +10229,47 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm24 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm14, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm13, %xmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm15, %xmm1
+; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm14, %xmm1
 ; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
-; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm1, %xmm1
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm28 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm28, %xmm1, %xmm1
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm28
 ; AVX512BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm28[0],xmm8[1],xmm28[1],xmm8[2],xmm28[2],xmm8[3],xmm28[3],xmm8[4],xmm28[4],xmm8[5],xmm28[5],xmm8[6],xmm28[6],xmm8[7],xmm28[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm7[0],xmm16[0],xmm7[1],xmm16[1],xmm7[2],xmm16[2],xmm7[3],xmm16[3],xmm7[4],xmm16[4],xmm7[5],xmm16[5],xmm7[6],xmm16[6],xmm7[7],xmm16[7]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[28,29,30],zero,ymm10[28],zero,ymm10[30,31,30,31],zero,ymm10[29],zero,ymm10[31,28,29]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[28,29,30],zero,ymm9[28],zero,ymm9[30,31,30,31],zero,ymm9[29],zero,ymm9[31,28,29]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm10[30],zero,ymm10[28],zero,zero,zero,zero,ymm10[31],zero,ymm10[29],zero,zero,zero
 ; AVX512BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm31
 ; AVX512BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm31[0],xmm3[1],xmm31[1],xmm3[2],xmm31[2],xmm3[3],xmm31[3],xmm3[4],xmm31[4],xmm3[5],xmm31[5],xmm3[6],xmm31[6],xmm3[7],xmm31[7]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero,ymm13[29],zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm30 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %ymm23, %ymm30, %ymm23
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero
+; AVX512BW-FCP-NEXT:    vporq %ymm23, %ymm29, %ymm23
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[2,3,2,3],zmm1[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm30
-; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
-; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm29, %xmm0, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29]
-; AVX512BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm1
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm29
+; AVX512BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm30
+; AVX512BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm28, %xmm0, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm13[27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} ymm28 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero,ymm14[29]
+; AVX512BW-FCP-NEXT:    vporq %ymm1, %ymm28, %ymm1
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -10278,82 +10278,82 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
 ; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm0
 ; AVX512BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm1
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm28 = zmm0[4,5,6,7],zmm1[4,5,6,7]
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm29[23],zero,zmm29[21,22,23,26],zero,zmm29[24],zero,zmm29[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm28[23],zero,zmm28[21,22,23,26],zero,zmm28[24],zero,zmm28[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm28[62],zero,zmm28[60],zero,zero,zero,zero,zmm28[63],zero,zmm28[61],zero
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61],zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm29
+; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm28
 ; AVX512BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm1
 ; AVX512BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm2
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm2[4,5,6,7],zmm1[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm6[60],zero,zmm6[62,63,62,63],zero,zmm6[61],zero,zmm6[63,60,61]
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm1[4,5,6,7],zmm2[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm7 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm7[23],zero,zero,zero,zero,zmm7[26],zero,zmm7[24],zero,zero,zero,zero,zmm7[27],zero,zmm7[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm7[62],zero,zmm7[60],zero,zero,zero,zero,zmm7[63],zero,zmm7[61],zero,zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm29[2,3,2,3,6,7,6,7]
-; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm2[4,5,6,7],zmm1[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm5[23],zero,zero,zero,zero,zmm5[26],zero,zmm5[24],zero,zero,zero,zero,zmm5[27],zero,zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm5[60],zero,zmm5[62,63,62,63],zero,zmm5[61],zero,zmm5[63,60,61]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm1[4,5,6,7],zmm2[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61],zero,zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm5, %zmm6, %zmm5
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm28[2,3,2,3,6,7,6,7]
+; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm28 = zmm5[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm29 {%k1}
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm6[4,5,6,7],zmm7[4,5,6,7]
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm7[4,5,6,7],zmm6[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm28 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm5
+; AVX512BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm6
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm5[4,5,6,7],zmm6[4,5,6,7]
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[4,5,6,7],zmm5[4,5,6,7]
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm6[25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61]
-; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm6, %zmm0
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm5[25],zero,zmm5[23],zero,zero,zero,zero,zmm5[26],zero,zmm5[24],zero,zero,zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm5[59],zero,zero,zero,zero,zmm5[62],zero,zmm5[60],zero,zero,zero,zero,zmm5[63],zero,zmm5[61]
+; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm5, %zmm0
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k1}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm28 {%k1}
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm0, %zmm0
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm0, %zmm0
 ; AVX512BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k2}
-; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm4, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm6
-; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm6, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm28 {%k2}
+; AVX512BW-FCP-NEXT:    vpshufb %xmm26, %xmm31, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm5
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm31[8],xmm3[8],xmm31[9],xmm3[9],xmm31[10],xmm3[10],xmm31[11],xmm3[11],xmm31[12],xmm3[12],xmm31[13],xmm3[13],xmm31[14],xmm3[14],xmm31[15],xmm3[15]
 ; AVX512BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm3
 ; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm28, %xmm3
-; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm8, %xmm4
-; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm28[8],xmm8[8],xmm28[9],xmm8[9],xmm28[10],xmm8[10],xmm28[11],xmm8[11],xmm28[12],xmm8[12],xmm28[13],xmm8[13],xmm28[14],xmm8[14],xmm28[15],xmm8[15]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm4
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm4[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm21, %xmm16, %xmm3
+; AVX512BW-FCP-NEXT:    vpshufb %xmm22, %xmm7, %xmm5
+; AVX512BW-FCP-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm16[8],xmm7[8],xmm16[9],xmm7[9],xmm16[10],xmm7[10],xmm16[11],xmm7[11],xmm16[12],xmm7[12],xmm16[13],xmm7[13],xmm16[14],xmm7[14],xmm16[15],xmm7[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm17, %xmm5, %xmm5
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm5[0,1,0,1]
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm30, %xmm0
-; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm31, %xmm4
-; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
-; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm4
-; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm4[0,1,0,1]
-; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm4, %zmm4
+; AVX512BW-FCP-NEXT:    vpshufb %xmm24, %xmm29, %xmm0
+; AVX512BW-FCP-NEXT:    vpshufb %xmm25, %xmm30, %xmm5
+; AVX512BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
+; AVX512BW-FCP-NEXT:    vpshufb %xmm20, %xmm5, %xmm5
+; AVX512BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm5[0,1,0,1]
+; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm5, %zmm5
 ; AVX512BW-FCP-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
+; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm5, %zmm0 {%k2}
 ; AVX512BW-FCP-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k2}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[18],zero,zmm12[18,19,20,21],zero,zmm12[19],zero,zmm12[25,26,27,22],zero,zmm12[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[56,57],zero,zmm12[55],zero,zmm12[53,54,55,58],zero,zmm12[56],zero,zmm12[60,61,58,59]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22],zero,zmm13[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero,zero,zero
-; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm4, %zmm0
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm1, %zmm1
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm11[18],zero,zmm11[18,19,20,21],zero,zmm11[19],zero,zmm11[25,26,27,22],zero,zmm11[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm11[56,57],zero,zmm11[55],zero,zmm11[53,54,55,58],zero,zmm11[56],zero,zmm11[60,61,58,59]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm12[18],zero,zero,zero,zero,zmm12[21],zero,zmm12[19],zero,zero,zero,zero,zmm12[22],zero,zmm12[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm12[57],zero,zmm12[55],zero,zero,zero,zero,zmm12[58],zero,zmm12[56],zero,zero,zero,zero
+; AVX512BW-FCP-NEXT:    vporq %zmm0, %zmm5, %zmm0
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm1, %zmm1
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero
-; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
 ; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[21],zero,zmm2[19],zero,zero,zero,zero,zmm2[22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zmm2[58],zero,zmm2[56],zero,zero,zero,zero,zmm2[59],zero,zmm2[57]
 ; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[20],zero,zmm15[18],zero,zmm15[20,21,20,21],zero,zmm15[19],zero,zmm15[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[56,57,56,57],zero,zmm15[55],zero,zmm15[55,56,57,58],zero,zmm15[56],zero,zmm15[62,63]
-; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm14[20],zero,zmm14[18],zero,zero,zero,zero,zmm14[21],zero,zmm14[19],zero,zero,zero,zero,zmm14[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm14[57],zero,zmm14[55],zero,zero,zero,zero,zmm14[58],zero,zmm14[56],zero,zero
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm14[20],zero,zmm14[18],zero,zmm14[20,21,20,21],zero,zmm14[19],zero,zmm14[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm14[56,57,56,57],zero,zmm14[55],zero,zmm14[55,56,57,58],zero,zmm14[56],zero,zmm14[62,63]
+; AVX512BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[20],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero
 ; AVX512BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
 ; AVX512BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512BW-FCP-NEXT:    vpermw %zmm16, %zmm2, %zmm2
+; AVX512BW-FCP-NEXT:    vpermw %zmm15, %zmm2, %zmm2
 ; AVX512BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512BW-FCP-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512BW-FCP-NEXT:    kmovq %rax, %k1
@@ -10363,12 +10363,12 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm29, 384(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm28, 384(%rax)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm23, 192(%rax)
 ; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm18, (%rax)
-; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512BW-FCP-NEXT:    vmovdqa64 %zmm8, 64(%rax)
 ; AVX512BW-FCP-NEXT:    vzeroupper
 ; AVX512BW-FCP-NEXT:    retq
 ;
@@ -10653,14 +10653,14 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-LABEL: store_i8_stride7_vf64:
 ; AVX512DQ-BW-FCP:       # %bb.0:
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm16
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rax), %zmm15
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm0, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm12, %ymm0, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm14 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm14, %ymm2, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm3, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm13 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm2, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm4, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,128,18,19,20,21,128,19,128,25,26,27,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,128,23,128,21,22,23,26,128,24,128,28,29,26,27]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,18,128,128,128,128,21,128,19,128,128,128,128,22,128,20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,25,128,23,128,128,128,128,26,128,24,128,128,128,128]
@@ -10670,105 +10670,106 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm5, %ymm6
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm6, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm1, %ymm1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm5, %ymm5
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm5, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm1, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rsi), %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm6, %ymm5
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm4, %ymm5, %ymm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,18,19,20,21,128,19,128,21,20,21,22,128,20,128,22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,21,128,19,128,128,128,128,22,128,20,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,23,128,128,128,128,26,128,24,128,128,128,128,27,128,25]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm6, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm6, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm1, %zmm4, %zmm4
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %r10, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm5 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm3
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm4 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r9), %ymm9
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm0 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm3, %ymm6
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm9, %ymm6
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%r8), %ymm7
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} ymm1 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm8
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm8, %ymm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm7, %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm3, %ymm10, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm7, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm6, %ymm10, %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,20,128,18,128,20,21,20,21,128,19,128,19,20,21,22,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,128,24,25,24,25,128,23,128,23,24,25,26,128,24,128,30,31]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm7, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,20,128,18,128,128,128,128,21,128,19,128,128,128,128,22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,128,128,128,128,25,128,23,128,128,128,128,26,128,24,128,128]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm3, %ymm9, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm11, %ymm9, %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[2,3,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm10, %zmm9
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rax), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm4, %ymm11
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} ymm11 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %ymm10, %ymm11, %ymm11
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31]
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm10, %zmm10
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $145249953336295682, %rax # imm = 0x204081020408102
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm3 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm10, %zmm9 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-4357498600088870461, %rax # imm = 0xC3870E1C3870E1C3
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm3, %zmm5 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm10
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm10, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm11
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm9, %ymm11, %ymm3
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm3, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm8
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm18
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm18[8],xmm3[8],xmm18[9],xmm3[9],xmm18[10],xmm3[10],xmm18[11],xmm3[11],xmm18[12],xmm3[12],xmm18[13],xmm3[13],xmm18[14],xmm3[14],xmm18[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm9, %zmm4 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rdi), %ymm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm2, %ymm9, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa (%rsi), %ymm10
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm8, %ymm10, %ymm8
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm8, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %xmm18
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdi), %xmm7
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %xmm23
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm16
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm8 = xmm23[8],xmm18[8],xmm23[9],xmm18[9],xmm23[10],xmm18[10],xmm23[11],xmm18[11],xmm23[12],xmm18[12],xmm23[13],xmm18[13],xmm23[14],xmm18[14],xmm23[15],xmm18[15]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm17 = [2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm9, %xmm9
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm9, %zmm9
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm8, %xmm8
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm8, %zmm8
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm12, %ymm11, %ymm2
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm14, %ymm13, %ymm14
-; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm14, %ymm2
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm14 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm13, %ymm12, %ymm13
+; AVX512DQ-BW-FCP-NEXT:    vpor %ymm2, %ymm13, %ymm2
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm13 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm19 = [6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm14, %xmm14
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm14, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm13, %xmm13
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $435749860008887046, %rax # imm = 0x60C183060C18306
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm9 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm2, %zmm8 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm13, %ymm0
 ; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm0, %ymm14, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm15, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %ymm1, %ymm14, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm15[8],xmm14[8],xmm15[9],xmm14[9],xmm15[10],xmm14[10],xmm15[11],xmm14[11],xmm15[12],xmm14[12],xmm15[13],xmm14[13],xmm15[14],xmm14[14],xmm15[15],xmm14[15]
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm1 = xmm14[8],xmm13[8],xmm14[9],xmm13[9],xmm14[10],xmm13[10],xmm14[11],xmm13[11],xmm14[12],xmm13[12],xmm14[13],xmm13[13],xmm14[14],xmm13[14],xmm14[15],xmm13[15]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm20 = [u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6,6,1,6,0,7,1,0,7,14,9,14,8,15,9,8,15]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $2323999253380730912, %rax # imm = 0x2040810204081020
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $4066998693416279096, %rax # imm = 0x3870E1C3870E1C38
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm9 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm8 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm21 = [u,128,7,128,5,u,u,u,128,8,128,6,u,u,u,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm18, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm23, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm22 = [u,7,128,5,128,u,u,u,8,128,6,128,u,u,u,9]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm3, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm18, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm18[0],xmm3[1],xmm18[1],xmm3[2],xmm18[2],xmm3[3],xmm18[3],xmm3[4],xmm18[4],xmm3[5],xmm18[5],xmm3[6],xmm18[6],xmm3[7],xmm18[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm18[0],xmm23[0],xmm18[1],xmm23[1],xmm18[2],xmm23[2],xmm18[3],xmm23[3],xmm18[4],xmm23[4],xmm18[5],xmm23[5],xmm18[6],xmm23[6],xmm18[7],xmm23[7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm1[0,1,0,1],zmm0[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm26 = [u,u,u,128,7,128,5,u,u,u,128,8,128,6,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm13, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm12, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm27 = [u,u,u,7,128,5,128,u,u,u,8,128,6,128,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm12, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm11, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm12[0],xmm13[0],xmm12[1],xmm13[1],xmm12[2],xmm13[2],xmm12[3],xmm13[3],xmm12[4],xmm13[4],xmm12[5],xmm13[5],xmm12[6],xmm13[6],xmm12[7],xmm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm11[0],xmm12[0],xmm11[1],xmm12[1],xmm11[2],xmm12[2],xmm11[3],xmm12[3],xmm11[4],xmm12[4],xmm11[5],xmm12[5],xmm11[6],xmm12[6],xmm11[7],xmm12[7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm23 = [4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
@@ -10776,48 +10777,47 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm24 = [128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm14, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm13, %xmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm25 = [4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm15, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm14, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm15[0],xmm14[0],xmm15[1],xmm14[1],xmm15[2],xmm14[2],xmm15[3],xmm14[3],xmm15[4],xmm14[4],xmm15[5],xmm14[5],xmm15[6],xmm14[6],xmm15[7],xmm14[7]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm29 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm1, %xmm1
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 {{.*#+}} xmm28 = [u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm28, %xmm1, %xmm1
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,0,1],zmm0[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $4647998506761461824, %rax # imm = 0x4081020408102040
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rsi), %xmm28
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $8133997386832558192, %rax # imm = 0x70E1C3870E1C3870
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm18 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm8[0],xmm28[0],xmm8[1],xmm28[1],xmm8[2],xmm28[2],xmm8[3],xmm28[3],xmm8[4],xmm28[4],xmm8[5],xmm28[5],xmm8[6],xmm28[6],xmm8[7],xmm28[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm7[0],xmm16[0],xmm7[1],xmm16[1],xmm7[2],xmm16[2],xmm7[3],xmm16[3],xmm7[4],xmm16[4],xmm7[5],xmm16[5],xmm7[6],xmm16[6],xmm7[7],xmm16[7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[28,29,30],zero,ymm10[28],zero,ymm10[30,31,30,31],zero,ymm10[29],zero,ymm10[31,28,29]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[28,29,30],zero,ymm9[28],zero,ymm9[30,31,30,31],zero,ymm9[29],zero,ymm9[31,28,29]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm2 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm10[30],zero,ymm10[28],zero,zero,zero,zero,ymm10[31],zero,ymm10[29],zero,zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vpor %ymm1, %ymm2, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rcx), %xmm4
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%rcx), %xmm31
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa 32(%rdx), %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm1 = xmm3[0],xmm31[0],xmm3[1],xmm31[1],xmm3[2],xmm31[2],xmm3[3],xmm31[3],xmm3[4],xmm31[4],xmm3[5],xmm31[5],xmm3[6],xmm31[6],xmm3[7],xmm31[7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm23, %xmm1, %xmm1
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero,ymm13[29],zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm30 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm23, %ymm30, %ymm23
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm23 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[30],zero,ymm11[28],zero,zero,zero,zero,ymm11[31],zero,ymm11[29],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm23, %ymm29, %ymm23
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm23[2,3,2,3],zmm1[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm30
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm31
-; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm30[0],xmm31[1],xmm30[1],xmm31[2],xmm30[2],xmm31[3],xmm30[3],xmm31[4],xmm30[4],xmm31[5],xmm30[5],xmm31[6],xmm30[6],xmm31[7],xmm30[7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm29, %xmm0, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm29 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero,ymm15[29]
-; AVX512DQ-BW-FCP-NEXT:    vporq %ymm1, %ymm29, %ymm1
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r9), %xmm29
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 32(%r8), %xmm30
+; AVX512DQ-BW-FCP-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm30[0],xmm29[0],xmm30[1],xmm29[1],xmm30[2],xmm29[2],xmm30[3],xmm29[3],xmm30[4],xmm29[4],xmm30[5],xmm29[5],xmm30[6],xmm29[6],xmm30[7],xmm29[7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm28, %xmm0, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm1 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm13[27],zero,zero,zero,zero,ymm13[30],zero,ymm13[28],zero,zero,zero,zero,ymm13[31],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} ymm28 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[27],zero,zero,zero,zero,ymm14[30],zero,ymm14[28],zero,zero,zero,zero,ymm14[31],zero,ymm14[29]
+; AVX512DQ-BW-FCP-NEXT:    vporq %ymm1, %ymm28, %ymm1
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[2,3,2,3],zmm0[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm1 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $580999813345182728, %rax # imm = 0x810204081020408
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
@@ -10826,82 +10826,82 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm23 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdx), %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rcx), %zmm1
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm29 = zmm0[4,5,6,7],zmm1[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm28 = zmm0[4,5,6,7],zmm1[4,5,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[4,5,6,7],zmm0[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm29[23],zero,zmm29[21,22,23,26],zero,zmm29[24],zero,zmm29[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm28[23],zero,zmm28[21,22,23,26],zero,zmm28[24],zero,zmm28[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm28[62],zero,zmm28[60],zero,zero,zero,zero,zmm28[63],zero,zmm28[61],zero
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm0[25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero,zmm0[61],zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm29
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm0, %zmm28
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rdi), %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%rsi), %zmm2
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm2[4,5,6,7],zmm1[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm6[60],zero,zmm6[62,63,62,63],zero,zmm6[61],zero,zmm6[63,60,61]
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm1[4,5,6,7],zmm2[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm7 = zmm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm7[23],zero,zero,zero,zero,zmm7[26],zero,zmm7[24],zero,zero,zero,zero,zmm7[27],zero,zmm7[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm7[62],zero,zmm7[60],zero,zero,zero,zero,zmm7[63],zero,zmm7[61],zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm6, %zmm7, %zmm6
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm7 = zmm29[2,3,2,3,6,7,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm29 = zmm6[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm2[4,5,6,7],zmm1[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm5[23],zero,zero,zero,zero,zmm5[26],zero,zmm5[24],zero,zero,zero,zero,zmm5[27],zero,zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm5[60],zero,zmm5[62,63,62,63],zero,zmm5[61],zero,zmm5[63,60,61]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm1[4,5,6,7],zmm2[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zero,zero,zmm6[27],zero,zmm6[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61],zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm5, %zmm6, %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm6 = zmm28[2,3,2,3,6,7,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm28 = zmm5[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm7, %zmm29 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm6[4,5,6,7],zmm7[4,5,6,7]
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm6 = zmm7[4,5,6,7],zmm6[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm6, %zmm28 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r8), %zmm5
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 (%r9), %zmm6
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm5[4,5,6,7],zmm6[4,5,6,7]
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[4,5,6,7],zmm5[4,5,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm6 = zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm6[25],zero,zmm6[23],zero,zero,zero,zero,zmm6[26],zero,zmm6[24],zero,zero,zmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm6[59],zero,zero,zero,zero,zmm6[62],zero,zmm6[60],zero,zero,zero,zero,zmm6[63],zero,zmm6[61]
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm6, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm5[25],zero,zmm5[23],zero,zero,zero,zero,zmm5[26],zero,zmm5[24],zero,zero,zmm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm5[59],zero,zero,zero,zero,zmm5[62],zero,zmm5[60],zero,zero,zero,zero,zmm5[63],zero,zmm5[61]
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm5, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k1}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm28 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm0 = [27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28,30,29,30,31,31,30,30,31,30,29,30,31,31,30,30,31]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm0, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm0, %zmm0
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm29 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm4, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm6
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm6, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm28 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm26, %xmm31, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm27, %xmm3, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm31[8],xmm3[8],xmm31[9],xmm3[9],xmm31[10],xmm3[10],xmm31[11],xmm3[11],xmm31[12],xmm3[12],xmm31[13],xmm3[13],xmm31[14],xmm3[14],xmm31[15],xmm3[15]
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm19, %xmm3, %xmm3
 ; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm3[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm28, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm8, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm28[8],xmm8[8],xmm28[9],xmm8[9],xmm28[10],xmm8[10],xmm28[11],xmm8[11],xmm28[12],xmm8[12],xmm28[13],xmm8[13],xmm28[14],xmm8[14],xmm28[15],xmm8[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm4[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm21, %xmm16, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm22, %xmm7, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm16[8],xmm7[8],xmm16[9],xmm7[9],xmm16[10],xmm7[10],xmm16[11],xmm7[11],xmm16[12],xmm7[12],xmm16[13],xmm7[13],xmm16[14],xmm7[14],xmm16[15],xmm7[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm17, %xmm5, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,0,1],zmm5[0,1,0,1]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm30, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm31, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm4, %xmm0
-; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm31[8],xmm30[8],xmm31[9],xmm30[9],xmm31[10],xmm30[10],xmm31[11],xmm30[11],xmm31[12],xmm30[12],xmm31[13],xmm30[13],xmm31[14],xmm30[14],xmm31[15],xmm30[15]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm4, %xmm4
-; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm4[0,1,0,1]
-; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm4 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm4, %zmm4
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm24, %xmm29, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm25, %xmm30, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vpor %xmm0, %xmm5, %xmm0
+; AVX512DQ-BW-FCP-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm30[8],xmm29[8],xmm30[9],xmm29[9],xmm30[10],xmm29[10],xmm30[11],xmm29[11],xmm30[12],xmm29[12],xmm30[13],xmm29[13],xmm30[14],xmm29[14],xmm30[15],xmm29[15]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb %xmm20, %xmm5, %xmm5
+; AVX512DQ-BW-FCP-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,0,1],zmm5[0,1,0,1]
+; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm5 = [18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20,20,21,20,21,21,22,21,22,20,21,20,21,21,22,21,22]
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm5, %zmm5
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $290499906672591364, %rax # imm = 0x408102040810204
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
-; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm4, %zmm0 {%k2}
+; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm5, %zmm0 {%k2}
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k2
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm0, %zmm3 {%k2}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[18],zero,zmm12[18,19,20,21],zero,zmm12[19],zero,zmm12[25,26,27,22],zero,zmm12[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm12[56,57],zero,zmm12[55],zero,zmm12[53,54,55,58],zero,zmm12[56],zero,zmm12[60,61,58,59]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm4 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22],zero,zmm13[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero,zero,zero
-; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm4, %zmm0
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm11, %zmm1, %zmm1
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm11[18],zero,zmm11[18,19,20,21],zero,zmm11[19],zero,zmm11[25,26,27,22],zero,zmm11[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm11[56,57],zero,zmm11[55],zero,zmm11[53,54,55,58],zero,zmm11[56],zero,zmm11[60,61,58,59]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm5 = zmm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm12[18],zero,zero,zero,zero,zmm12[21],zero,zmm12[19],zero,zero,zero,zero,zmm12[22],zero,zmm12[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm12[57],zero,zmm12[55],zero,zero,zero,zero,zmm12[58],zero,zmm12[56],zero,zero,zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vporq %zmm0, %zmm5, %zmm0
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm1, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm1[18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero
-; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm2[21],zero,zmm2[19],zero,zero,zero,zero,zmm2[22],zero,zmm2[20],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm2[55],zero,zero,zero,zero,zmm2[58],zero,zmm2[56],zero,zero,zero,zero,zmm2[59],zero,zmm2[57]
 ; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[20],zero,zmm15[18],zero,zmm15[20,21,20,21],zero,zmm15[19],zero,zmm15[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm15[56,57,56,57],zero,zmm15[55],zero,zmm15[55,56,57,58],zero,zmm15[56],zero,zmm15[62,63]
-; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm14[20],zero,zmm14[18],zero,zero,zero,zero,zmm14[21],zero,zmm14[19],zero,zero,zero,zero,zmm14[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm14[57],zero,zmm14[55],zero,zero,zero,zero,zmm14[58],zero,zmm14[56],zero,zero
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm14[20],zero,zmm14[18],zero,zmm14[20,21,20,21],zero,zmm14[19],zero,zmm14[19,20,21,22],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zmm14[56,57,56,57],zero,zmm14[55],zero,zmm14[55,56,57,58],zero,zmm14[56],zero,zmm14[62,63]
+; AVX512DQ-BW-FCP-NEXT:    vpshufb {{.*#+}} zmm2 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[20],zero,zmm13[18],zero,zero,zero,zero,zmm13[21],zero,zmm13[19],zero,zero,zero,zero,zmm13[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm13[57],zero,zmm13[55],zero,zero,zero,zero,zmm13[58],zero,zmm13[56],zero,zero
 ; AVX512DQ-BW-FCP-NEXT:    vporq %zmm1, %zmm2, %zmm1
 ; AVX512DQ-BW-FCP-NEXT:    vpmovsxbw {{.*#+}} zmm2 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12]
-; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm16, %zmm2, %zmm2
+; AVX512DQ-BW-FCP-NEXT:    vpermw %zmm15, %zmm2, %zmm2
 ; AVX512DQ-BW-FCP-NEXT:    vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7]
 ; AVX512DQ-BW-FCP-NEXT:    movabsq $1161999626690365456, %rax # imm = 0x1020408102040810
 ; AVX512DQ-BW-FCP-NEXT:    kmovq %rax, %k1
@@ -10911,12 +10911,12 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqu8 %zmm1, %zmm0 {%k1}
 ; AVX512DQ-BW-FCP-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm4, 320(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm29, 384(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm28, 384(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm23, 192(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm18, (%rax)
-; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512DQ-BW-FCP-NEXT:    vmovdqa64 %zmm8, 64(%rax)
 ; AVX512DQ-BW-FCP-NEXT:    vzeroupper
 ; AVX512DQ-BW-FCP-NEXT:    retq
   %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64
-- 
cgit v1.2.3


From ad36800d65250618b1c1446ae0f97d250bf09df8 Mon Sep 17 00:00:00 2001
From: NAKAMURA Takumi <geek4civic@gmail.com>
Date: Sat, 21 Jun 2025 07:43:57 +0900
Subject: [bazel][lld] Follow up 491b82a5ec1a
 (llvmorg-21-init-16192-g491b82a5ec1a)

---
 utils/bazel/llvm-project-overlay/lld/BUILD.bazel | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
index 450157758d75..6533d783c2b3 100644
--- a/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/lld/BUILD.bazel
@@ -74,6 +74,7 @@ cc_library(
         "ELF/*.cpp",
         "ELF/*.h",
         "ELF/Arch/*.cpp",
+        "ELF/Arch/*.h",
     ]),
     includes = ["ELF"],
     textual_hdrs = [
-- 
cgit v1.2.3


From be7e4113c84317feda055cac68a855429e46e381 Mon Sep 17 00:00:00 2001
From: Shilei Tian <i@tianshilei.me>
Date: Fri, 20 Jun 2025 18:55:41 -0400
Subject: [NFC] Add comment to describe the intention use of newly added
 `avail-extern-gv-in-addrspace-to-local` (#144911)

---
 llvm/lib/Transforms/IPO/ElimAvailExtern.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp b/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp
index bc98f994f490..538755e66f9c 100644
--- a/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp
+++ b/llvm/lib/Transforms/IPO/ElimAvailExtern.cpp
@@ -35,6 +35,10 @@ static cl::opt<bool> ConvertToLocal(
     cl::desc("Convert available_externally into locals, renaming them "
              "to avoid link-time clashes."));
 
+// This option was originally introduced to correctly support the lowering of
+// LDS variables for AMDGPU when ThinLTO is enabled. It can be utilized for
+// other purposes, but make sure it is safe to do so, as privatizing global
+// variables is generally not safe.
 static cl::opt<unsigned> ConvertGlobalVariableInAddrSpace(
     "avail-extern-gv-in-addrspace-to-local", cl::Hidden,
     cl::desc(
-- 
cgit v1.2.3


From 6b9fe9e0bc7421d89f38ade857c91f0ae153ad9c Mon Sep 17 00:00:00 2001
From: Justin Bogner <mail@justinbogner.com>
Date: Fri, 20 Jun 2025 16:21:47 -0700
Subject: [HLSL] Emit a version in the dx.rootsignatures metadata (#145113)

In #144957 the backend was updated to expect a version in the metadata,
but since the frontend wasn't updated this breaks compilation. This is a
somewhat temporary fix to that until #144813 lands.
---
 clang/lib/CodeGen/CGHLSLRuntime.cpp                          | 11 ++++++++---
 clang/test/CodeGenHLSL/RootSignature.hlsl                    | 12 ++++++------
 llvm/lib/Target/DirectX/DXILRootSignature.cpp                |  4 ++--
 .../CodeGen/DirectX/ContainerData/RootSignature-Error.ll     |  2 +-
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp
index 34960c34e109..3103f1798e14 100644
--- a/clang/lib/CodeGen/CGHLSLRuntime.cpp
+++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp
@@ -72,12 +72,17 @@ void addRootSignature(ArrayRef<llvm::hlsl::rootsig::RootElement> Elements,
 
   llvm::hlsl::rootsig::MetadataBuilder Builder(Ctx, Elements);
   MDNode *RootSignature = Builder.BuildRootSignature();
-  MDNode *FnPairing =
-      MDNode::get(Ctx, {ValueAsMetadata::get(Fn), RootSignature});
+
+  // TODO: We need to wire the root signature version up through the frontend
+  // rather than hardcoding it.
+  ConstantAsMetadata *Version =
+      ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 2));
+  MDNode *MDVals =
+      MDNode::get(Ctx, {ValueAsMetadata::get(Fn), RootSignature, Version});
 
   StringRef RootSignatureValKey = "dx.rootsignatures";
   auto *RootSignatureValMD = M.getOrInsertNamedMetadata(RootSignatureValKey);
-  RootSignatureValMD->addOperand(FnPairing);
+  RootSignatureValMD->addOperand(MDVals);
 }
 
 } // namespace
diff --git a/clang/test/CodeGenHLSL/RootSignature.hlsl b/clang/test/CodeGenHLSL/RootSignature.hlsl
index ca843ffbb1ce..6618ca741aa9 100644
--- a/clang/test/CodeGenHLSL/RootSignature.hlsl
+++ b/clang/test/CodeGenHLSL/RootSignature.hlsl
@@ -3,14 +3,14 @@
 // CHECK: !dx.rootsignatures = !{![[#EMPTY_ENTRY:]], ![[#DT_ENTRY:]],
 // CHECK-SAME: ![[#RF_ENTRY:]], ![[#RC_ENTRY:]], ![[#RD_ENTRY:]], ![[#SS_ENTRY:]]}
 
-// CHECK: ![[#EMPTY_ENTRY]] = !{ptr @EmptyEntry, ![[#EMPTY:]]}
+// CHECK: ![[#EMPTY_ENTRY]] = !{ptr @EmptyEntry, ![[#EMPTY:]], i32 2}
 // CHECK: ![[#EMPTY]] = !{}
 
 [shader("compute"), RootSignature("")]
 [numthreads(1,1,1)]
 void EmptyEntry() {}
 
-// CHECK: ![[#DT_ENTRY]] = !{ptr @DescriptorTableEntry, ![[#DT_RS:]]}
+// CHECK: ![[#DT_ENTRY]] = !{ptr @DescriptorTableEntry, ![[#DT_RS:]], i32 2}
 // CHECK: ![[#DT_RS]] = !{![[#TABLE:]]}
 // CHECK: ![[#TABLE]] = !{!"DescriptorTable", i32 0, ![[#CBV:]], ![[#SRV:]]}
 // CHECK: ![[#CBV]] = !{!"CBV", i32 1, i32 0, i32 0, i32 -1, i32 4}
@@ -25,7 +25,7 @@ void EmptyEntry() {}
 [numthreads(1,1,1)]
 void DescriptorTableEntry() {}
 
-// CHECK: ![[#RF_ENTRY]] = !{ptr @RootFlagsEntry, ![[#RF_RS:]]}
+// CHECK: ![[#RF_ENTRY]] = !{ptr @RootFlagsEntry, ![[#RF_RS:]], i32 2}
 // CHECK: ![[#RF_RS]] = !{![[#ROOT_FLAGS:]]}
 // CHECK: ![[#ROOT_FLAGS]] = !{!"RootFlags", i32 2114}
 
@@ -38,7 +38,7 @@ void DescriptorTableEntry() {}
 [numthreads(1,1,1)]
 void RootFlagsEntry() {}
 
-// CHECK: ![[#RC_ENTRY]] = !{ptr @RootConstantsEntry, ![[#RC_RS:]]}
+// CHECK: ![[#RC_ENTRY]] = !{ptr @RootConstantsEntry, ![[#RC_RS:]], i32 2}
 // CHECK: ![[#RC_RS]] = !{![[#ROOT_CONSTANTS:]]}
 // CHECK: ![[#ROOT_CONSTANTS]] = !{!"RootConstants", i32 5, i32 1, i32 2, i32 1}
 
@@ -52,7 +52,7 @@ void RootFlagsEntry() {}
 [numthreads(1,1,1)]
 void RootConstantsEntry() {}
 
-// CHECK: ![[#RD_ENTRY]] = !{ptr @RootDescriptorsEntry, ![[#RD_RS:]]}
+// CHECK: ![[#RD_ENTRY]] = !{ptr @RootDescriptorsEntry, ![[#RD_RS:]], i32 2}
 // CHECK: ![[#RD_RS]] = !{![[#ROOT_CBV:]], ![[#ROOT_UAV:]], ![[#ROOT_SRV:]]}
 // CHECK: ![[#ROOT_CBV]] = !{!"RootCBV", i32 0, i32 0, i32 0, i32 4}
 // CHECK: ![[#ROOT_UAV]] = !{!"RootUAV", i32 0, i32 42, i32 3, i32 2}
@@ -66,7 +66,7 @@ void RootConstantsEntry() {}
 [numthreads(1,1,1)]
 void RootDescriptorsEntry() {}
 
-// CHECK: ![[#SS_ENTRY]] = !{ptr @StaticSamplerEntry, ![[#SS_RS:]]}
+// CHECK: ![[#SS_ENTRY]] = !{ptr @StaticSamplerEntry, ![[#SS_RS:]], i32 2}
 // CHECK: ![[#SS_RS]] = !{![[#STATIC_SAMPLER:]]}
 
 // checking filter = 0x4
diff --git a/llvm/lib/Target/DirectX/DXILRootSignature.cpp b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
index 1f175fd4ecd9..29e78fcce526 100644
--- a/llvm/lib/Target/DirectX/DXILRootSignature.cpp
+++ b/llvm/lib/Target/DirectX/DXILRootSignature.cpp
@@ -491,8 +491,8 @@ analyzeModule(Module &M) {
 
   for (const auto &RSDefNode : RootSignatureNode->operands()) {
     if (RSDefNode->getNumOperands() != 3) {
-      reportError(Ctx, "Invalid format for Root Signature Definition. Pairs "
-                       "of function, root signature expected.");
+      reportError(Ctx, "Invalid Root Signature metadata - expected function, "
+                       "signature, and version.");
       continue;
     }
 
diff --git a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error.ll b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error.ll
index 2a2188b1a13b..039206584211 100644
--- a/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error.ll
+++ b/llvm/test/CodeGen/DirectX/ContainerData/RootSignature-Error.ll
@@ -2,7 +2,7 @@
 
 target triple = "dxil-unknown-shadermodel6.0-compute"
 
-; CHECK: error: Invalid format for Root Signature Definition. Pairs of function, root signature expected.
+; CHECK: error: Invalid Root Signature metadata - expected function, signature, and version.
 ; CHECK-NOT: Root Signature Definitions
 
 
-- 
cgit v1.2.3


From 2dfcc4375faa5e3692bd82a022d33bdd6fe55f10 Mon Sep 17 00:00:00 2001
From: Erick Velez <erickvelez7@gmail.com>
Date: Fri, 20 Jun 2025 16:38:20 -0700
Subject: [clang-doc] Precommit concept tests (#144160)

---
 .../test/clang-doc/json/class-requires.cpp         | 34 ++++++++++
 clang-tools-extra/test/clang-doc/json/concept.cpp  | 37 ++++++++++
 .../test/clang-doc/json/function-requires.cpp      | 79 ++++++++++++++++++++++
 3 files changed, 150 insertions(+)
 create mode 100644 clang-tools-extra/test/clang-doc/json/class-requires.cpp
 create mode 100644 clang-tools-extra/test/clang-doc/json/concept.cpp
 create mode 100644 clang-tools-extra/test/clang-doc/json/function-requires.cpp

diff --git a/clang-tools-extra/test/clang-doc/json/class-requires.cpp b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
new file mode 100644
index 000000000000..af108a402b40
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
@@ -0,0 +1,34 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/MyClass.json
+
+template<typename T>
+concept Addable = requires(T a, T b) {
+  { a + b };
+};
+
+template<typename T>
+requires Addable<T>
+struct MyClass;
+
+// CHECK:       "Name": "MyClass",
+// CHECK-NEXT:  "Namespace": [
+// CHECK-NEXT:    "GlobalNamespace"
+// CHECK-NEXT:  ],
+// CHECK-NEXT:  "Path": "GlobalNamespace",
+// CHECK-NEXT:  "TagType": "struct",
+// CHECK-NEXT:  "Template": {
+// CHECK-NOT:     "Constraints": [
+// CHECK-NOT:       {
+// CHECK-NOT:         "Expression": "Addable<T>",
+// CHECK-NOT:         "Name": "Addable",
+// CHECK-NOT:         "Path": "",
+// CHECK-NOT:         "QualName": "Addable",
+// CHECK-NOT:         "USR": "{{[0-9A-F]*}}"
+// CHECK-NOT:       }
+// CHECK-NOT:     ],
+// CHECK-NEXT:    "Parameters": [
+// CHECK-NEXT:      "typename T"
+// CHECK-NEXT:    ]
+// CHECK-NEXT:  },
+// CHECK-NEXT:  "USR": "{{[0-9A-F]*}}"
diff --git a/clang-tools-extra/test/clang-doc/json/concept.cpp b/clang-tools-extra/test/clang-doc/json/concept.cpp
new file mode 100644
index 000000000000..624f71c6bf9b
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/concept.cpp
@@ -0,0 +1,37 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
+
+// Requires that T suports post and pre-incrementing.
+template<typename T>
+concept Incrementable = requires(T x) {
+  ++x;
+  x++;
+};
+
+// CHECK:      {
+// CHECK-NOT:    "Concepts": [
+// CHECK-NOT:      {
+// CHECK-NOT:        "ConstraintExpression": "requires (T x) { ++x; x++; }",
+// CHECK-NOT:        "Description": [
+// CHECK-NOT:          {
+// CHECK-NOT:            "FullComment": {
+// CHECK-NOT:              "Children": [
+// CHECK-NOT:                {
+// CHECK-NOT:                  "ParagraphComment": {
+// CHECK-NOT:                    "Children": [
+// CHECK-NOT:                      {
+// CHECK-NOT:                        "TextComment": " Requires that T suports post and pre-incrementing."
+// CHECK-NOT:        ],
+// CHECK-NOT:        "IsType": true,
+// CHECK-NOT:        "Name": "Incrementable",
+// CHECK-NOT:        "Template": {
+// CHECK-NOT:          "Parameters": [
+// CHECK-NOT:            "typename T"
+// CHECK-NOT:          ]
+// CHECK-NOT:        },
+// CHECK-NOT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NOT:      }
+// CHECK-NOT:    ],
+// CHECK:        "Name": "",
+// CHECK:        "USR": "0000000000000000000000000000000000000000"
+// CHECK:      }
diff --git a/clang-tools-extra/test/clang-doc/json/function-requires.cpp b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
new file mode 100644
index 000000000000..aa62464d07b4
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
@@ -0,0 +1,79 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/index.json
+
+template<typename T>
+concept Incrementable = requires(T x) {
+  ++x;
+  x++;
+};
+
+template<typename T> void increment(T t) requires Incrementable<T>;
+
+template<Incrementable T> Incrementable auto incrementTwo(T t);
+
+// CHECK:       "Functions": [
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "IsStatic": false,
+// CHECK-NEXT:      "Name": "increment",
+// CHECK-NEXT:      "Params": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Name": "t",
+// CHECK-NEXT:          "Type": "T"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "ReturnType": {
+// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsTemplate": false,
+// CHECK-NEXT:        "Name": "void",
+// CHECK-NEXT:        "QualName": "void",
+// CHECK-NEXT:        "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "Template": {
+// CHECK-NOT:         "Constraints": [
+// CHECK-NOT:           {
+// CHECK-NOT:             "Expression": "Incrementable<T>",
+// CHECK-NOT:             "Name": "Incrementable",
+// CHECK-NOT:             "Path": "",
+// CHECK-NOT:             "QualName": "Incrementable",
+// CHECK-NOT:             "USR": "{{[0-9A-F]*}}"
+// CHECK-NOT:           }
+// CHECK-NOT:         ],
+// CHECK-NEXT:        "Parameters": [
+// CHECK-NEXT:          "typename T"
+// CHECK-NEXT:        ]
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "USR": "{{[0-9A-F]*}}" 
+// CHECK-NEXT:    },
+// CHECK-NEXT:    {
+// CHECK-NEXT:      "IsStatic": false,
+// CHECK-NEXT:      "Name": "incrementTwo",
+// CHECK-NEXT:      "Params": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Name": "t",
+// CHECK-NEXT:          "Type": "T"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
+// CHECK-NEXT:      "ReturnType": {
+// CHECK-NEXT:        "IsBuiltIn": false,
+// CHECK-NEXT:        "IsTemplate": false,
+// CHECK-NEXT:        "Name": "Incrementable auto",
+// CHECK-NEXT:        "QualName": "Incrementable auto",
+// CHECK-NEXT:        "USR": "0000000000000000000000000000000000000000"
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "Template": {
+// CHECK-NOT:         "Constraints": [
+// CHECK-NOT:           {
+// CHECK-NOT:             "Expression": "Incrementable<T>",
+// CHECK-NOT:             "Name": "Incrementable",
+// CHECK-NOT:             "Path": "",
+// CHECK-NOT:             "QualName": "Incrementable",
+// CHECK-NOT:             "USR": "{{[0-9A-F]*}}"
+// CHECK-NOT:           }
+// CHECK-NOT:         ],
+// CHECK-NEXT:        "Parameters": [
+// CHECK-NEXT:          "Incrementable T"
+// CHECK-NEXT:        ]
+// CHECK-NEXT:      },
+// CHECK-NEXT:      "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:    }
-- 
cgit v1.2.3


From 8050a6e0732c6614ce3e5296fdeb5a3c36bde26d Mon Sep 17 00:00:00 2001
From: Erick Velez <erickvelez7@gmail.com>
Date: Fri, 20 Jun 2025 17:39:31 -0700
Subject: [clang-doc] add support for concepts (#144430)

Add support for documenting concepts. This handles concepts and constraints on function and class templates.

Atomic constraints are not considered yet. We don't order constraints based on their conjunctive or disjunctive properties.
---
 clang-tools-extra/clang-doc/BitcodeReader.cpp      |  73 +++++++++++++
 clang-tools-extra/clang-doc/BitcodeWriter.cpp      |  44 +++++++-
 clang-tools-extra/clang-doc/BitcodeWriter.h        |  12 +-
 clang-tools-extra/clang-doc/HTMLGenerator.cpp      |   4 +
 .../clang-doc/HTMLMustacheGenerator.cpp            |   2 +
 clang-tools-extra/clang-doc/JSONGenerator.cpp      |  50 +++++++++
 clang-tools-extra/clang-doc/MDGenerator.cpp        |   5 +
 clang-tools-extra/clang-doc/Mapper.cpp             |   4 +
 clang-tools-extra/clang-doc/Mapper.h               |   1 +
 clang-tools-extra/clang-doc/Representation.cpp     |  20 ++++
 clang-tools-extra/clang-doc/Representation.h       |  26 ++++-
 clang-tools-extra/clang-doc/Serialize.cpp          |  90 +++++++++++++++
 clang-tools-extra/clang-doc/Serialize.h            |   4 +
 clang-tools-extra/clang-doc/YAMLGenerator.cpp      |   2 +
 .../test/clang-doc/json/class-requires.cpp         |  18 +--
 .../test/clang-doc/json/compound-constraints.cpp   | 121 +++++++++++++++++++++
 clang-tools-extra/test/clang-doc/json/concept.cpp  |  49 +++++----
 .../test/clang-doc/json/function-requires.cpp      |  36 +++---
 .../unittests/clang-doc/BitcodeTest.cpp            |   2 +
 19 files changed, 507 insertions(+), 56 deletions(-)
 create mode 100644 clang-tools-extra/test/clang-doc/json/compound-constraints.cpp

diff --git a/clang-tools-extra/clang-doc/BitcodeReader.cpp b/clang-tools-extra/clang-doc/BitcodeReader.cpp
index 35058abab066..66852931226b 100644
--- a/clang-tools-extra/clang-doc/BitcodeReader.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeReader.cpp
@@ -92,6 +92,7 @@ static llvm::Error decodeRecord(const Record &R, InfoType &Field,
   case InfoType::IT_default:
   case InfoType::IT_enum:
   case InfoType::IT_typedef:
+  case InfoType::IT_concept:
     Field = IT;
     return llvm::Error::success();
   }
@@ -108,6 +109,7 @@ static llvm::Error decodeRecord(const Record &R, FieldId &Field,
   case FieldId::F_type:
   case FieldId::F_child_namespace:
   case FieldId::F_child_record:
+  case FieldId::F_concept:
   case FieldId::F_default:
     Field = F;
     return llvm::Error::success();
@@ -391,6 +393,29 @@ static llvm::Error parseRecord(const Record &R, unsigned ID,
                                  "invalid field for TemplateParamInfo");
 }
 
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, ConceptInfo *I) {
+  switch (ID) {
+  case CONCEPT_USR:
+    return decodeRecord(R, I->USR, Blob);
+  case CONCEPT_NAME:
+    return decodeRecord(R, I->Name, Blob);
+  case CONCEPT_IS_TYPE:
+    return decodeRecord(R, I->IsType, Blob);
+  case CONCEPT_CONSTRAINT_EXPRESSION:
+    return decodeRecord(R, I->ConstraintExpression, Blob);
+  }
+  llvm_unreachable("invalid field for ConceptInfo");
+}
+
+static llvm::Error parseRecord(const Record &R, unsigned ID,
+                               llvm::StringRef Blob, ConstraintInfo *I) {
+  if (ID == CONSTRAINT_EXPRESSION)
+    return decodeRecord(R, I->ConstraintExpr, Blob);
+  return llvm::createStringError(llvm::inconvertibleErrorCode(),
+                                 "invalid field for ConstraintInfo");
+}
+
 template <typename T> static llvm::Expected<CommentInfo *> getCommentInfo(T I) {
   return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                  "invalid type cannot contain CommentInfo");
@@ -429,6 +454,10 @@ template <> llvm::Expected<CommentInfo *> getCommentInfo(CommentInfo *I) {
   return I->Children.back().get();
 }
 
+template <> llvm::Expected<CommentInfo *> getCommentInfo(ConceptInfo *I) {
+  return &I->Description.emplace_back();
+}
+
 // When readSubBlock encounters a TypeInfo sub-block, it calls addTypeInfo on
 // the parent block to set it. The template specializations define what to do
 // for each supported parent block.
@@ -584,6 +613,17 @@ template <> llvm::Error addReference(RecordInfo *I, Reference &&R, FieldId F) {
   }
 }
 
+template <>
+llvm::Error addReference(ConstraintInfo *I, Reference &&R, FieldId F) {
+  if (F == FieldId::F_concept) {
+    I->ConceptRef = std::move(R);
+    return llvm::Error::success();
+  }
+  return llvm::createStringError(
+      llvm::inconvertibleErrorCode(),
+      "ConstraintInfo cannot contain this Reference");
+}
+
 template <typename T, typename ChildInfoType>
 static void addChild(T I, ChildInfoType &&R) {
   llvm::errs() << "invalid child type for info";
@@ -600,6 +640,9 @@ template <> void addChild(NamespaceInfo *I, EnumInfo &&R) {
 template <> void addChild(NamespaceInfo *I, TypedefInfo &&R) {
   I->Children.Typedefs.emplace_back(std::move(R));
 }
+template <> void addChild(NamespaceInfo *I, ConceptInfo &&R) {
+  I->Children.Concepts.emplace_back(std::move(R));
+}
 
 // Record children:
 template <> void addChild(RecordInfo *I, FunctionInfo &&R) {
@@ -649,6 +692,9 @@ template <> void addTemplate(RecordInfo *I, TemplateInfo &&P) {
 template <> void addTemplate(FunctionInfo *I, TemplateInfo &&P) {
   I->Template.emplace(std::move(P));
 }
+template <> void addTemplate(ConceptInfo *I, TemplateInfo &&P) {
+  I->Template = std::move(P);
+}
 
 // Template specializations go only into template records.
 template <typename T>
@@ -662,6 +708,14 @@ void addTemplateSpecialization(TemplateInfo *I,
   I->Specialization.emplace(std::move(TSI));
 }
 
+template <typename T> static void addConstraint(T I, ConstraintInfo &&C) {
+  llvm::errs() << "invalid container for constraint info";
+  exit(1);
+}
+template <> void addConstraint(TemplateInfo *I, ConstraintInfo &&C) {
+  I->Constraints.emplace_back(std::move(C));
+}
+
 // Read records from bitcode into a given info.
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readRecord(unsigned ID, T I) {
@@ -716,6 +770,8 @@ llvm::Error ClangDocBitcodeReader::readBlock(unsigned ID, T I) {
   }
 }
 
+// TODO: Create a helper that can receive a function to reduce repetition for
+// most blocks.
 template <typename T>
 llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
   llvm::TimeTraceScope("Reducing infos", "readSubBlock");
@@ -817,6 +873,20 @@ llvm::Error ClangDocBitcodeReader::readSubBlock(unsigned ID, T I) {
     addChild(I, std::move(TI));
     return llvm::Error::success();
   }
+  case BI_CONSTRAINT_BLOCK_ID: {
+    ConstraintInfo CI;
+    if (auto Err = readBlock(ID, &CI))
+      return Err;
+    addConstraint(I, std::move(CI));
+    return llvm::Error::success();
+  }
+  case BI_CONCEPT_BLOCK_ID: {
+    ConceptInfo CI;
+    if (auto Err = readBlock(ID, &CI))
+      return Err;
+    addChild(I, std::move(CI));
+    return llvm::Error::success();
+  }
   default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "invalid subblock type");
@@ -922,6 +992,8 @@ ClangDocBitcodeReader::readBlockToInfo(unsigned ID) {
     return createInfo<EnumInfo>(ID);
   case BI_TYPEDEF_BLOCK_ID:
     return createInfo<TypedefInfo>(ID);
+  case BI_CONCEPT_BLOCK_ID:
+    return createInfo<ConceptInfo>(ID);
   case BI_FUNCTION_BLOCK_ID:
     return createInfo<FunctionInfo>(ID);
   default:
@@ -962,6 +1034,7 @@ ClangDocBitcodeReader::readBitcode() {
     case BI_RECORD_BLOCK_ID:
     case BI_ENUM_BLOCK_ID:
     case BI_TYPEDEF_BLOCK_ID:
+    case BI_CONCEPT_BLOCK_ID:
     case BI_FUNCTION_BLOCK_ID: {
       auto InfoOrErr = readBlockToInfo(ID);
       if (!InfoOrErr)
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.cpp b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
index f8a6859169b0..b7308c012786 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.cpp
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.cpp
@@ -128,7 +128,9 @@ static const llvm::IndexedMap<llvm::StringRef, BlockIdToIndexFunctor>
           {BI_REFERENCE_BLOCK_ID, "ReferenceBlock"},
           {BI_TEMPLATE_BLOCK_ID, "TemplateBlock"},
           {BI_TEMPLATE_SPECIALIZATION_BLOCK_ID, "TemplateSpecializationBlock"},
-          {BI_TEMPLATE_PARAM_BLOCK_ID, "TemplateParamBlock"}};
+          {BI_TEMPLATE_PARAM_BLOCK_ID, "TemplateParamBlock"},
+          {BI_CONSTRAINT_BLOCK_ID, "ConstraintBlock"},
+          {BI_CONCEPT_BLOCK_ID, "ConceptBlock"}};
       assert(Inits.size() == BlockIdCount);
       for (const auto &Init : Inits)
         BlockIdNameMap[Init.first] = Init.second;
@@ -205,7 +207,13 @@ static const llvm::IndexedMap<RecordIdDsc, RecordIdToIndexFunctor>
           {TYPEDEF_USR, {"USR", &genSymbolIdAbbrev}},
           {TYPEDEF_NAME, {"Name", &genStringAbbrev}},
           {TYPEDEF_DEFLOCATION, {"DefLocation", &genLocationAbbrev}},
-          {TYPEDEF_IS_USING, {"IsUsing", &genBoolAbbrev}}};
+          {TYPEDEF_IS_USING, {"IsUsing", &genBoolAbbrev}},
+          {CONCEPT_USR, {"USR", &genSymbolIdAbbrev}},
+          {CONCEPT_NAME, {"Name", &genStringAbbrev}},
+          {CONCEPT_IS_TYPE, {"IsType", &genBoolAbbrev}},
+          {CONCEPT_CONSTRAINT_EXPRESSION,
+           {"ConstraintExpression", &genStringAbbrev}},
+          {CONSTRAINT_EXPRESSION, {"Expression", &genStringAbbrev}}};
       assert(Inits.size() == RecordIdCount);
       for (const auto &Init : Inits) {
         RecordIdNameMap[Init.first] = Init.second;
@@ -263,7 +271,13 @@ static const std::vector<std::pair<BlockId, std::vector<RecordId>>>
         // Template Blocks.
         {BI_TEMPLATE_BLOCK_ID, {}},
         {BI_TEMPLATE_PARAM_BLOCK_ID, {TEMPLATE_PARAM_CONTENTS}},
-        {BI_TEMPLATE_SPECIALIZATION_BLOCK_ID, {TEMPLATE_SPECIALIZATION_OF}}};
+        {BI_TEMPLATE_SPECIALIZATION_BLOCK_ID, {TEMPLATE_SPECIALIZATION_OF}},
+        // Concept Block
+        {BI_CONCEPT_BLOCK_ID,
+         {CONCEPT_USR, CONCEPT_NAME, CONCEPT_IS_TYPE,
+          CONCEPT_CONSTRAINT_EXPRESSION}},
+        // Constraint Block
+        {BI_CONSTRAINT_BLOCK_ID, {CONSTRAINT_EXPRESSION}}};
 
 // AbbreviationMap
 
@@ -524,6 +538,8 @@ void ClangDocBitcodeWriter::emitBlock(const NamespaceInfo &I) {
     emitBlock(C);
   for (const auto &C : I.Children.Typedefs)
     emitBlock(C);
+  for (const auto &C : I.Children.Concepts)
+    emitBlock(C);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const EnumInfo &I) {
@@ -627,12 +643,25 @@ void ClangDocBitcodeWriter::emitBlock(const FunctionInfo &I) {
     emitBlock(*I.Template);
 }
 
+void ClangDocBitcodeWriter::emitBlock(const ConceptInfo &I) {
+  StreamSubBlockGuard Block(Stream, BI_CONCEPT_BLOCK_ID);
+  emitRecord(I.USR, CONCEPT_USR);
+  emitRecord(I.Name, CONCEPT_NAME);
+  for (const auto &CI : I.Description)
+    emitBlock(CI);
+  emitRecord(I.IsType, CONCEPT_IS_TYPE);
+  emitRecord(I.ConstraintExpression, CONCEPT_CONSTRAINT_EXPRESSION);
+  emitBlock(I.Template);
+}
+
 void ClangDocBitcodeWriter::emitBlock(const TemplateInfo &T) {
   StreamSubBlockGuard Block(Stream, BI_TEMPLATE_BLOCK_ID);
   for (const auto &P : T.Params)
     emitBlock(P);
   if (T.Specialization)
     emitBlock(*T.Specialization);
+  for (const auto &C : T.Constraints)
+    emitBlock(C);
 }
 
 void ClangDocBitcodeWriter::emitBlock(const TemplateSpecializationInfo &T) {
@@ -647,6 +676,12 @@ void ClangDocBitcodeWriter::emitBlock(const TemplateParamInfo &T) {
   emitRecord(T.Contents, TEMPLATE_PARAM_CONTENTS);
 }
 
+void ClangDocBitcodeWriter::emitBlock(const ConstraintInfo &C) {
+  StreamSubBlockGuard Block(Stream, BI_CONSTRAINT_BLOCK_ID);
+  emitRecord(C.ConstraintExpr, CONSTRAINT_EXPRESSION);
+  emitBlock(C.ConceptRef, FieldId::F_concept);
+}
+
 bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   switch (I->IT) {
   case InfoType::IT_namespace:
@@ -664,6 +699,9 @@ bool ClangDocBitcodeWriter::dispatchInfoForWrite(Info *I) {
   case InfoType::IT_typedef:
     emitBlock(*static_cast<clang::doc::TypedefInfo *>(I));
     break;
+  case InfoType::IT_concept:
+    emitBlock(*static_cast<clang::doc::ConceptInfo *>(I));
+    break;
   case InfoType::IT_default:
     llvm::errs() << "Unexpected info, unable to write.\n";
     return true;
diff --git a/clang-tools-extra/clang-doc/BitcodeWriter.h b/clang-tools-extra/clang-doc/BitcodeWriter.h
index e33a1aece883..4d0c0c07805e 100644
--- a/clang-tools-extra/clang-doc/BitcodeWriter.h
+++ b/clang-tools-extra/clang-doc/BitcodeWriter.h
@@ -66,7 +66,9 @@ enum BlockId {
   BI_TEMPLATE_BLOCK_ID,
   BI_TEMPLATE_SPECIALIZATION_BLOCK_ID,
   BI_TEMPLATE_PARAM_BLOCK_ID,
+  BI_CONSTRAINT_BLOCK_ID,
   BI_TYPEDEF_BLOCK_ID,
+  BI_CONCEPT_BLOCK_ID,
   BI_LAST,
   BI_FIRST = BI_VERSION_BLOCK_ID
 };
@@ -135,6 +137,11 @@ enum RecordId {
   TYPEDEF_NAME,
   TYPEDEF_DEFLOCATION,
   TYPEDEF_IS_USING,
+  CONCEPT_USR,
+  CONCEPT_NAME,
+  CONCEPT_IS_TYPE,
+  CONCEPT_CONSTRAINT_EXPRESSION,
+  CONSTRAINT_EXPRESSION,
   RI_LAST,
   RI_FIRST = VERSION
 };
@@ -150,7 +157,8 @@ enum class FieldId {
   F_vparent,
   F_type,
   F_child_namespace,
-  F_child_record
+  F_child_record,
+  F_concept
 };
 
 class ClangDocBitcodeWriter {
@@ -179,6 +187,8 @@ public:
   void emitBlock(const TemplateInfo &T);
   void emitBlock(const TemplateSpecializationInfo &T);
   void emitBlock(const TemplateParamInfo &T);
+  void emitBlock(const ConceptInfo &T);
+  void emitBlock(const ConstraintInfo &T);
   void emitBlock(const Reference &B, FieldId F);
 
 private:
diff --git a/clang-tools-extra/clang-doc/HTMLGenerator.cpp b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
index 7293a129177c..935bbfee7a9b 100644
--- a/clang-tools-extra/clang-doc/HTMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLGenerator.cpp
@@ -985,6 +985,8 @@ llvm::Error HTMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
     MainContentNodes =
         genHTML(*static_cast<clang::doc::TypedefInfo *>(I), CDCtx, InfoTitle);
     break;
+  case InfoType::IT_concept:
+    break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected info type");
@@ -1011,6 +1013,8 @@ static std::string getRefType(InfoType IT) {
     return "enum";
   case InfoType::IT_typedef:
     return "typedef";
+  case InfoType::IT_concept:
+    return "concept";
   }
   llvm_unreachable("Unknown InfoType");
 }
diff --git a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
index 69c670b20844..81ba99c21e37 100644
--- a/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
+++ b/clang-tools-extra/clang-doc/HTMLMustacheGenerator.cpp
@@ -585,6 +585,8 @@ Error MustacheHTMLGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
   case InfoType::IT_typedef:
     OS << "IT_typedef\n";
     break;
+  case InfoType::IT_concept:
+    break;
   case InfoType::IT_default:
     return createStringError(inconvertibleErrorCode(), "unexpected InfoType");
   }
diff --git a/clang-tools-extra/clang-doc/JSONGenerator.cpp b/clang-tools-extra/clang-doc/JSONGenerator.cpp
index 0f7cbafcf513..8a37621597c6 100644
--- a/clang-tools-extra/clang-doc/JSONGenerator.cpp
+++ b/clang-tools-extra/clang-doc/JSONGenerator.cpp
@@ -26,6 +26,15 @@ static void serializeInfo(const TypedefInfo &I, json::Object &Obj,
                           std::optional<StringRef> RepositoryUrl);
 static void serializeInfo(const EnumInfo &I, json::Object &Obj,
                           std::optional<StringRef> RepositoryUrl);
+static void serializeInfo(const ConstraintInfo &I, Object &Obj);
+
+// Convenience lambda to pass to serializeArray.
+// If a serializeInfo needs a RepositoryUrl, create a local lambda that captures
+// the optional.
+static auto SerializeInfoLambda = [](const ConstraintInfo &Info,
+                                     Object &Object) {
+  serializeInfo(Info, Object);
+};
 
 static json::Object serializeLocation(const Location &Loc,
                                       std::optional<StringRef> RepositoryUrl) {
@@ -248,6 +257,27 @@ static void serializeCommonChildren(const ScopeChildren &Children,
   }
 }
 
+template <typename T, typename SerializationFunc>
+static void serializeArray(const std::vector<T> &Records, Object &Obj,
+                           const std::string &Key,
+                           SerializationFunc SerializeInfo) {
+  json::Value RecordsArray = Array();
+  auto &RecordsArrayRef = *RecordsArray.getAsArray();
+  RecordsArrayRef.reserve(Records.size());
+  for (const auto &Item : Records) {
+    json::Value ItemVal = Object();
+    auto &ItemObj = *ItemVal.getAsObject();
+    SerializeInfo(Item, ItemObj);
+    RecordsArrayRef.push_back(ItemVal);
+  }
+  Obj[Key] = RecordsArray;
+}
+
+static void serializeInfo(const ConstraintInfo &I, Object &Obj) {
+  serializeReference(I.ConceptRef, Obj);
+  Obj["Expression"] = I.ConstraintExpr;
+}
+
 static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
   json::Value TemplateVal = Object();
   auto &TemplateObj = *TemplateVal.getAsObject();
@@ -277,9 +307,21 @@ static void serializeInfo(const TemplateInfo &Template, Object &Obj) {
     TemplateObj["Parameters"] = ParamsArray;
   }
 
+  if (!Template.Constraints.empty())
+    serializeArray(Template.Constraints, TemplateObj, "Constraints",
+                   SerializeInfoLambda);
+
   Obj["Template"] = TemplateVal;
 }
 
+static void serializeInfo(const ConceptInfo &I, Object &Obj,
+                          std::optional<StringRef> RepositoryUrl) {
+  serializeCommonAttributes(I, Obj, RepositoryUrl);
+  Obj["IsType"] = I.IsType;
+  Obj["ConstraintExpression"] = I.ConstraintExpression;
+  serializeInfo(I.Template, Obj);
+}
+
 static void serializeInfo(const TypeInfo &I, Object &Obj) {
   Obj["Name"] = I.Type.Name;
   Obj["QualName"] = I.Type.QualName;
@@ -457,6 +499,10 @@ static void serializeInfo(const NamespaceInfo &I, json::Object &Obj,
     Obj["Namespaces"] = NamespacesArray;
   }
 
+  auto SerializeInfo = [RepositoryUrl](const auto &Info, Object &Object) {
+    serializeInfo(Info, Object, RepositoryUrl);
+  };
+
   if (!I.Children.Functions.empty()) {
     json::Value FunctionsArray = Array();
     auto &FunctionsArrayRef = *FunctionsArray.getAsArray();
@@ -470,6 +516,9 @@ static void serializeInfo(const NamespaceInfo &I, json::Object &Obj,
     Obj["Functions"] = FunctionsArray;
   }
 
+  if (!I.Children.Concepts.empty())
+    serializeArray(I.Children.Concepts, Obj, "Concepts", SerializeInfo);
+
   serializeCommonChildren(I.Children, Obj, RepositoryUrl);
 }
 
@@ -520,6 +569,7 @@ Error JSONGenerator::generateDocForInfo(Info *I, raw_ostream &OS,
   case InfoType::IT_record:
     serializeInfo(*static_cast<RecordInfo *>(I), Obj, CDCtx.RepositoryUrl);
     break;
+  case InfoType::IT_concept:
   case InfoType::IT_enum:
   case InfoType::IT_function:
   case InfoType::IT_typedef:
diff --git a/clang-tools-extra/clang-doc/MDGenerator.cpp b/clang-tools-extra/clang-doc/MDGenerator.cpp
index 2becccf8b07d..6e68e09cfa2a 100644
--- a/clang-tools-extra/clang-doc/MDGenerator.cpp
+++ b/clang-tools-extra/clang-doc/MDGenerator.cpp
@@ -372,6 +372,9 @@ static llvm::Error genIndex(ClangDocContext &CDCtx) {
       case InfoType::IT_typedef:
         Type = "Typedef";
         break;
+      case InfoType::IT_concept:
+        Type = "Concept";
+        break;
       case InfoType::IT_default:
         Type = "Other";
       }
@@ -464,6 +467,8 @@ llvm::Error MDGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
   case InfoType::IT_typedef:
     genMarkdown(CDCtx, *static_cast<clang::doc::TypedefInfo *>(I), OS);
     break;
+  case InfoType::IT_concept:
+    break;
   case InfoType::IT_default:
     return createStringError(llvm::inconvertibleErrorCode(),
                              "unexpected InfoType");
diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp
index 9f640b5325da..6021e17b4696 100644
--- a/clang-tools-extra/clang-doc/Mapper.cpp
+++ b/clang-tools-extra/clang-doc/Mapper.cpp
@@ -134,6 +134,10 @@ bool MapASTVisitor::VisitTypeAliasDecl(const TypeAliasDecl *D) {
   return mapDecl(D, /*isDefinition=*/true);
 }
 
+bool MapASTVisitor::VisitConceptDecl(const ConceptDecl *D) {
+  return mapDecl(D, true);
+}
+
 comments::FullComment *
 MapASTVisitor::getComment(const NamedDecl *D, const ASTContext &Context) const {
   RawComment *Comment = Context.getRawCommentForDeclNoCache(D);
diff --git a/clang-tools-extra/clang-doc/Mapper.h b/clang-tools-extra/clang-doc/Mapper.h
index 36322ea2bfb7..04dc5450c8ba 100644
--- a/clang-tools-extra/clang-doc/Mapper.h
+++ b/clang-tools-extra/clang-doc/Mapper.h
@@ -41,6 +41,7 @@ public:
   bool VisitFunctionDecl(const FunctionDecl *D);
   bool VisitTypedefDecl(const TypedefDecl *D);
   bool VisitTypeAliasDecl(const TypeAliasDecl *D);
+  bool VisitConceptDecl(const ConceptDecl *D);
 
 private:
   template <typename T> bool mapDecl(const T *D, bool IsDefinition);
diff --git a/clang-tools-extra/clang-doc/Representation.cpp b/clang-tools-extra/clang-doc/Representation.cpp
index 71a926f1c73e..286aeeea1001 100644
--- a/clang-tools-extra/clang-doc/Representation.cpp
+++ b/clang-tools-extra/clang-doc/Representation.cpp
@@ -143,6 +143,8 @@ mergeInfos(std::vector<std::unique_ptr<Info>> &Values) {
     return reduce<FunctionInfo>(Values);
   case InfoType::IT_typedef:
     return reduce<TypedefInfo>(Values);
+  case InfoType::IT_concept:
+    return reduce<ConceptInfo>(Values);
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected info type");
@@ -288,6 +290,7 @@ void NamespaceInfo::merge(NamespaceInfo &&Other) {
   reduceChildren(Children.Functions, std::move(Other.Children.Functions));
   reduceChildren(Children.Enums, std::move(Other.Children.Enums));
   reduceChildren(Children.Typedefs, std::move(Other.Children.Typedefs));
+  reduceChildren(Children.Concepts, std::move(Other.Children.Concepts));
   mergeBase(std::move(Other));
 }
 
@@ -352,6 +355,19 @@ void TypedefInfo::merge(TypedefInfo &&Other) {
   SymbolInfo::merge(std::move(Other));
 }
 
+void ConceptInfo::merge(ConceptInfo &&Other) {
+  assert(mergeable(Other));
+  if (!IsType)
+    IsType = Other.IsType;
+  if (ConstraintExpression.empty())
+    ConstraintExpression = std::move(Other.ConstraintExpression);
+  if (Template.Constraints.empty())
+    Template.Constraints = std::move(Other.Template.Constraints);
+  if (Template.Params.empty())
+    Template.Params = std::move(Other.Template.Params);
+  SymbolInfo::merge(std::move(Other));
+}
+
 BaseRecordInfo::BaseRecordInfo() : RecordInfo() {}
 
 BaseRecordInfo::BaseRecordInfo(SymbolID USR, StringRef Name, StringRef Path,
@@ -388,6 +404,9 @@ llvm::SmallString<16> Info::extractName() const {
   case InfoType::IT_function:
     return llvm::SmallString<16>("@nonymous_function_" +
                                  toHex(llvm::toStringRef(USR)));
+  case InfoType::IT_concept:
+    return llvm::SmallString<16>("@nonymous_concept_" +
+                                 toHex(llvm::toStringRef(USR)));
   case InfoType::IT_default:
     return llvm::SmallString<16>("@nonymous_" + toHex(llvm::toStringRef(USR)));
   }
@@ -453,6 +472,7 @@ void ScopeChildren::sort() {
   llvm::sort(Functions.begin(), Functions.end());
   llvm::sort(Enums.begin(), Enums.end());
   llvm::sort(Typedefs.begin(), Typedefs.end());
+  llvm::sort(Concepts.begin(), Concepts.end());
 }
 } // namespace doc
 } // namespace clang
diff --git a/clang-tools-extra/clang-doc/Representation.h b/clang-tools-extra/clang-doc/Representation.h
index 75da50064581..b23069f2bd32 100644
--- a/clang-tools-extra/clang-doc/Representation.h
+++ b/clang-tools-extra/clang-doc/Representation.h
@@ -35,6 +35,7 @@ struct EnumInfo;
 struct FunctionInfo;
 struct Info;
 struct TypedefInfo;
+struct ConceptInfo;
 
 enum class InfoType {
   IT_default,
@@ -42,7 +43,8 @@ enum class InfoType {
   IT_record,
   IT_function,
   IT_enum,
-  IT_typedef
+  IT_typedef,
+  IT_concept
 };
 
 enum class CommentKind {
@@ -166,6 +168,7 @@ struct ScopeChildren {
   std::vector<FunctionInfo> Functions;
   std::vector<EnumInfo> Enums;
   std::vector<TypedefInfo> Typedefs;
+  std::vector<ConceptInfo> Concepts;
 
   void sort();
 };
@@ -211,6 +214,15 @@ struct TemplateSpecializationInfo {
   std::vector<TemplateParamInfo> Params;
 };
 
+struct ConstraintInfo {
+  ConstraintInfo() = default;
+  ConstraintInfo(SymbolID USR, StringRef Name)
+      : ConceptRef(USR, Name, InfoType::IT_concept) {}
+  Reference ConceptRef;
+
+  SmallString<16> ConstraintExpr;
+};
+
 // Records the template information for a struct or function that is a template
 // or an explicit template specialization.
 struct TemplateInfo {
@@ -219,6 +231,7 @@ struct TemplateInfo {
 
   // Set when this is a specialization of another record/function.
   std::optional<TemplateSpecializationInfo> Specialization;
+  std::vector<ConstraintInfo> Constraints;
 };
 
 // Info for field types.
@@ -513,6 +526,17 @@ struct EnumInfo : public SymbolInfo {
   llvm::SmallVector<EnumValueInfo, 4> Members; // List of enum members.
 };
 
+struct ConceptInfo : public SymbolInfo {
+  ConceptInfo() : SymbolInfo(InfoType::IT_concept) {}
+  ConceptInfo(SymbolID USR) : SymbolInfo(InfoType::IT_concept, USR) {}
+
+  void merge(ConceptInfo &&I);
+
+  bool IsType;
+  TemplateInfo Template;
+  SmallString<16> ConstraintExpression;
+};
+
 struct Index : public Reference {
   Index() = default;
   Index(StringRef Name) : Reference(SymbolID(), Name) {}
diff --git a/clang-tools-extra/clang-doc/Serialize.cpp b/clang-tools-extra/clang-doc/Serialize.cpp
index 820e8bfd8e64..5f3e5c37fa34 100644
--- a/clang-tools-extra/clang-doc/Serialize.cpp
+++ b/clang-tools-extra/clang-doc/Serialize.cpp
@@ -21,6 +21,17 @@ namespace clang {
 namespace doc {
 namespace serialize {
 
+namespace {
+static SmallString<16> exprToString(const clang::Expr *E) {
+  clang::LangOptions Opts;
+  clang::PrintingPolicy Policy(Opts);
+  SmallString<16> Result;
+  llvm::raw_svector_ostream OS(Result);
+  E->printPretty(OS, nullptr, Policy);
+  return Result;
+}
+} // namespace
+
 SymbolID hashUSR(llvm::StringRef USR) {
   return llvm::SHA1::hash(arrayRefFromStringRef(USR));
 }
@@ -388,6 +399,8 @@ std::string serialize(std::unique_ptr<Info> &I) {
     return serialize(*static_cast<EnumInfo *>(I.get()));
   case InfoType::IT_function:
     return serialize(*static_cast<FunctionInfo *>(I.get()));
+  case InfoType::IT_concept:
+    return serialize(*static_cast<ConceptInfo *>(I.get()));
   case InfoType::IT_typedef:
   case InfoType::IT_default:
     return "";
@@ -491,6 +504,10 @@ static void InsertChild(ScopeChildren &Scope, TypedefInfo Info) {
   Scope.Typedefs.push_back(std::move(Info));
 }
 
+static void InsertChild(ScopeChildren &Scope, ConceptInfo Info) {
+  Scope.Concepts.push_back(std::move(Info));
+}
+
 // Creates a parent of the correct type for the given child and inserts it into
 // that parent.
 //
@@ -531,6 +548,7 @@ static std::unique_ptr<Info> makeAndInsertIntoParent(ChildType Child) {
   case InfoType::IT_enum:
   case InfoType::IT_function:
   case InfoType::IT_typedef:
+  case InfoType::IT_concept:
     break;
   }
   llvm_unreachable("Invalid reference type for parent namespace");
@@ -740,6 +758,50 @@ static void populateSymbolInfo(SymbolInfo &I, const T *D, const FullComment *C,
     I.Loc.emplace_back(Loc);
 }
 
+static void
+handleCompoundConstraints(const Expr *Constraint,
+                          std::vector<ConstraintInfo> &ConstraintInfos) {
+  if (Constraint->getStmtClass() == Stmt::ParenExprClass) {
+    handleCompoundConstraints(dyn_cast<ParenExpr>(Constraint)->getSubExpr(),
+                              ConstraintInfos);
+  } else if (Constraint->getStmtClass() == Stmt::BinaryOperatorClass) {
+    auto *BinaryOpExpr = dyn_cast<BinaryOperator>(Constraint);
+    handleCompoundConstraints(BinaryOpExpr->getLHS(), ConstraintInfos);
+    handleCompoundConstraints(BinaryOpExpr->getRHS(), ConstraintInfos);
+  } else if (Constraint->getStmtClass() ==
+             Stmt::ConceptSpecializationExprClass) {
+    auto *Concept = dyn_cast<ConceptSpecializationExpr>(Constraint);
+    ConstraintInfo CI(getUSRForDecl(Concept->getNamedConcept()),
+                      Concept->getNamedConcept()->getNameAsString());
+    CI.ConstraintExpr = exprToString(Concept);
+    ConstraintInfos.push_back(CI);
+  }
+}
+
+static void populateConstraints(TemplateInfo &I, const TemplateDecl *D) {
+  if (!D || !D->hasAssociatedConstraints())
+    return;
+
+  SmallVector<AssociatedConstraint> AssociatedConstraints;
+  D->getAssociatedConstraints(AssociatedConstraints);
+  for (const auto &Constraint : AssociatedConstraints) {
+    if (!Constraint)
+      continue;
+
+    // TODO: Investigate if atomic constraints need to be handled specifically.
+    if (const auto *ConstraintExpr =
+            dyn_cast_or_null<ConceptSpecializationExpr>(
+                Constraint.ConstraintExpr)) {
+      ConstraintInfo CI(getUSRForDecl(ConstraintExpr->getNamedConcept()),
+                        ConstraintExpr->getNamedConcept()->getNameAsString());
+      CI.ConstraintExpr = exprToString(ConstraintExpr);
+      I.Constraints.push_back(std::move(CI));
+    } else {
+      handleCompoundConstraints(Constraint.ConstraintExpr, I.Constraints);
+    }
+  }
+}
+
 static void populateFunctionInfo(FunctionInfo &I, const FunctionDecl *D,
                                  const FullComment *FC, Location Loc,
                                  bool &IsInAnonymousNamespace) {
@@ -751,6 +813,8 @@ static void populateFunctionInfo(FunctionInfo &I, const FunctionDecl *D,
   I.IsStatic = D->isStatic();
 
   populateTemplateParameters(I.Template, D);
+  if (I.Template)
+    populateConstraints(I.Template.value(), D->getDescribedFunctionTemplate());
 
   // Handle function template specializations.
   if (const FunctionTemplateSpecializationInfo *FTSI =
@@ -903,6 +967,8 @@ emitInfo(const RecordDecl *D, const FullComment *FC, Location Loc,
   RI->Path = getInfoRelativePath(RI->Namespace);
 
   populateTemplateParameters(RI->Template, D);
+  if (RI->Template)
+    populateConstraints(RI->Template.value(), D->getDescribedTemplate());
 
   // Full and partial specializations.
   if (auto *CTSD = dyn_cast<ClassTemplateSpecializationDecl>(D)) {
@@ -1074,6 +1140,30 @@ emitInfo(const EnumDecl *D, const FullComment *FC, Location Loc,
   return {nullptr, makeAndInsertIntoParent<EnumInfo &&>(std::move(Enum))};
 }
 
+std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
+emitInfo(const ConceptDecl *D, const FullComment *FC, const Location &Loc,
+         bool PublicOnly) {
+  ConceptInfo Concept;
+
+  bool IsInAnonymousNamespace = false;
+  populateInfo(Concept, D, FC, IsInAnonymousNamespace);
+  Concept.IsType = D->isTypeConcept();
+  Concept.DefLoc = Loc;
+  Concept.ConstraintExpression = exprToString(D->getConstraintExpr());
+
+  if (auto *ConceptParams = D->getTemplateParameters()) {
+    for (const auto *Param : ConceptParams->asArray()) {
+      Concept.Template.Params.emplace_back(
+          getSourceCode(Param, Param->getSourceRange()));
+    }
+  }
+
+  if (!shouldSerializeInfo(PublicOnly, IsInAnonymousNamespace, D))
+    return {};
+
+  return {nullptr, makeAndInsertIntoParent<ConceptInfo &&>(std::move(Concept))};
+}
+
 } // namespace serialize
 } // namespace doc
 } // namespace clang
diff --git a/clang-tools-extra/clang-doc/Serialize.h b/clang-tools-extra/clang-doc/Serialize.h
index 7e6cbb70721e..497b09bb339f 100644
--- a/clang-tools-extra/clang-doc/Serialize.h
+++ b/clang-tools-extra/clang-doc/Serialize.h
@@ -68,6 +68,10 @@ std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
 emitInfo(const TypeAliasDecl *D, const FullComment *FC, Location Loc,
          bool PublicOnly);
 
+std::pair<std::unique_ptr<Info>, std::unique_ptr<Info>>
+emitInfo(const ConceptDecl *D, const FullComment *FC, const Location &Loc,
+         bool PublicOnly);
+
 // Function to hash a given USR value for storage.
 // As USRs (Unified Symbol Resolution) could be large, especially for functions
 // with long type arguments, we use 160-bits SHA1(USR) values to
diff --git a/clang-tools-extra/clang-doc/YAMLGenerator.cpp b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
index 897b5d5ae4c9..f95887104698 100644
--- a/clang-tools-extra/clang-doc/YAMLGenerator.cpp
+++ b/clang-tools-extra/clang-doc/YAMLGenerator.cpp
@@ -408,6 +408,8 @@ llvm::Error YAMLGenerator::generateDocForInfo(Info *I, llvm::raw_ostream &OS,
   case InfoType::IT_typedef:
     InfoYAML << *static_cast<clang::doc::TypedefInfo *>(I);
     break;
+  case InfoType::IT_concept:
+    break;
   case InfoType::IT_default:
     return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                    "unexpected InfoType");
diff --git a/clang-tools-extra/test/clang-doc/json/class-requires.cpp b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
index af108a402b40..2dd25771d6d8 100644
--- a/clang-tools-extra/test/clang-doc/json/class-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/class-requires.cpp
@@ -18,15 +18,15 @@ struct MyClass;
 // CHECK-NEXT:  "Path": "GlobalNamespace",
 // CHECK-NEXT:  "TagType": "struct",
 // CHECK-NEXT:  "Template": {
-// CHECK-NOT:     "Constraints": [
-// CHECK-NOT:       {
-// CHECK-NOT:         "Expression": "Addable<T>",
-// CHECK-NOT:         "Name": "Addable",
-// CHECK-NOT:         "Path": "",
-// CHECK-NOT:         "QualName": "Addable",
-// CHECK-NOT:         "USR": "{{[0-9A-F]*}}"
-// CHECK-NOT:       }
-// CHECK-NOT:     ],
+// CHECK-NEXT:    "Constraints": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "Expression": "Addable<T>",
+// CHECK-NEXT:        "Name": "Addable",
+// CHECK-NEXT:        "Path": "",
+// CHECK-NEXT:        "QualName": "Addable",
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
 // CHECK-NEXT:    "Parameters": [
 // CHECK-NEXT:      "typename T"
 // CHECK-NEXT:    ]
diff --git a/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp b/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp
new file mode 100644
index 000000000000..b49dec5cc78c
--- /dev/null
+++ b/clang-tools-extra/test/clang-doc/json/compound-constraints.cpp
@@ -0,0 +1,121 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/index.json
+
+template<typename T> concept Incrementable = requires (T a) {
+  a++;
+};
+
+template<typename T> concept Decrementable = requires (T a) {
+  a--;
+};
+
+template<typename T> concept PreIncrementable = requires (T a) {
+  ++a;
+};
+
+template<typename T> concept PreDecrementable = requires (T a) {
+  --a;
+};
+
+template<typename T> requires Incrementable<T> && Decrementable<T> void One();
+
+template<typename T> requires (Incrementable<T> && Decrementable<T>) void Two();
+
+template<typename T> requires (Incrementable<T> && Decrementable<T>) || (PreIncrementable<T> && PreDecrementable<T>) void Three();
+
+template<typename T> requires (Incrementable<T> && Decrementable<T>) || PreIncrementable<T> void Four();
+
+// CHECK:         "Name": "One",
+// CHECK:         "Template": {
+// CHECK-NEXT:      "Constraints": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Incrementable<T>",
+// CHECK-NEXT:          "Name": "Incrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Incrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Decrementable<T>",
+// CHECK-NEXT:          "Name": "Decrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Decrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
+// CHECK:         "Name": "Two",
+// CHECK:         "Template": {
+// CHECK-NEXT:      "Constraints": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Incrementable<T>",
+// CHECK-NEXT:          "Name": "Incrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Incrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Decrementable<T>",
+// CHECK-NEXT:          "Name": "Decrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Decrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
+// CHECK:         "Name": "Three",
+// CHECK:         "Template": {
+// CHECK-NEXT:      "Constraints": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Incrementable<T>",
+// CHECK-NEXT:          "Name": "Incrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Incrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Decrementable<T>",
+// CHECK-NEXT:          "Name": "Decrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Decrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "PreIncrementable<T>",
+// CHECK-NEXT:          "Name": "PreIncrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "PreIncrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "PreDecrementable<T>",
+// CHECK-NEXT:          "Name": "PreDecrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "PreDecrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
+// CHECK:         "Name": "Four",
+// CHECK:         "Template": {
+// CHECK-NEXT:      "Constraints": [
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Incrementable<T>",
+// CHECK-NEXT:          "Name": "Incrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Incrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "Decrementable<T>",
+// CHECK-NEXT:          "Name": "Decrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "Decrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        },
+// CHECK-NEXT:        {
+// CHECK-NEXT:          "Expression": "PreIncrementable<T>",
+// CHECK-NEXT:          "Name": "PreIncrementable",
+// CHECK-NEXT:          "Path": "",
+// CHECK-NEXT:          "QualName": "PreIncrementable",
+// CHECK-NEXT:          "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:        }
+// CHECK-NEXT:      ],
diff --git a/clang-tools-extra/test/clang-doc/json/concept.cpp b/clang-tools-extra/test/clang-doc/json/concept.cpp
index 624f71c6bf9b..887c9d79146a 100644
--- a/clang-tools-extra/test/clang-doc/json/concept.cpp
+++ b/clang-tools-extra/test/clang-doc/json/concept.cpp
@@ -1,5 +1,6 @@
 // RUN: rm -rf %t && mkdir -p %t
 // RUN: clang-doc --extra-arg -std=c++20 --output=%t --format=json --executor=standalone %s
+// RUN: FileCheck %s < %t/GlobalNamespace/index.json
 
 // Requires that T suports post and pre-incrementing.
 template<typename T>
@@ -8,30 +9,30 @@ concept Incrementable = requires(T x) {
   x++;
 };
 
-// CHECK:      {
-// CHECK-NOT:    "Concepts": [
-// CHECK-NOT:      {
-// CHECK-NOT:        "ConstraintExpression": "requires (T x) { ++x; x++; }",
-// CHECK-NOT:        "Description": [
-// CHECK-NOT:          {
-// CHECK-NOT:            "FullComment": {
-// CHECK-NOT:              "Children": [
-// CHECK-NOT:                {
-// CHECK-NOT:                  "ParagraphComment": {
-// CHECK-NOT:                    "Children": [
-// CHECK-NOT:                      {
-// CHECK-NOT:                        "TextComment": " Requires that T suports post and pre-incrementing."
-// CHECK-NOT:        ],
-// CHECK-NOT:        "IsType": true,
-// CHECK-NOT:        "Name": "Incrementable",
-// CHECK-NOT:        "Template": {
-// CHECK-NOT:          "Parameters": [
-// CHECK-NOT:            "typename T"
-// CHECK-NOT:          ]
-// CHECK-NOT:        },
-// CHECK-NOT:        "USR": "{{[0-9A-F]*}}"
-// CHECK-NOT:      }
-// CHECK-NOT:    ],
+// CHECK:       {
+// CHECK-NEXT:    "Concepts": [
+// CHECK-NEXT:      {
+// CHECK-NEXT:        "ConstraintExpression": "requires (T x) { ++x; x++; }",
+// CHECK-NEXT:        "Description": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "FullComment": {
+// CHECK-NEXT:              "Children": [
+// CHECK-NEXT:                {
+// CHECK-NEXT:                  "ParagraphComment": {
+// CHECK-NEXT:                    "Children": [
+// CHECK-NEXT:                      {
+// CHECK-NEXT:                        "TextComment": " Requires that T suports post and pre-incrementing."
+// CHECK:             ],
+// CHECK-NEXT:        "IsType": true,
+// CHECK-NEXT:        "Name": "Incrementable",
+// CHECK-NEXT:        "Template": {
+// CHECK-NEXT:          "Parameters": [
+// CHECK-NEXT:            "typename T"
+// CHECK-NEXT:          ]
+// CHECK-NEXT:        },
+// CHECK-NEXT:        "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:      }
+// CHECK-NEXT:    ],
 // CHECK:        "Name": "",
 // CHECK:        "USR": "0000000000000000000000000000000000000000"
 // CHECK:      }
diff --git a/clang-tools-extra/test/clang-doc/json/function-requires.cpp b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
index aa62464d07b4..99eb2bdb898f 100644
--- a/clang-tools-extra/test/clang-doc/json/function-requires.cpp
+++ b/clang-tools-extra/test/clang-doc/json/function-requires.cpp
@@ -30,15 +30,15 @@ template<Incrementable T> Incrementable auto incrementTwo(T t);
 // CHECK-NEXT:        "USR": "0000000000000000000000000000000000000000"
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "Template": {
-// CHECK-NOT:         "Constraints": [
-// CHECK-NOT:           {
-// CHECK-NOT:             "Expression": "Incrementable<T>",
-// CHECK-NOT:             "Name": "Incrementable",
-// CHECK-NOT:             "Path": "",
-// CHECK-NOT:             "QualName": "Incrementable",
-// CHECK-NOT:             "USR": "{{[0-9A-F]*}}"
-// CHECK-NOT:           }
-// CHECK-NOT:         ],
+// CHECK-NEXT:        "Constraints": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Expression": "Incrementable<T>",
+// CHECK-NEXT:            "Name": "Incrementable",
+// CHECK-NEXT:            "Path": "",
+// CHECK-NEXT:            "QualName": "Incrementable",
+// CHECK-NEXT:            "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
 // CHECK-NEXT:        "Parameters": [
 // CHECK-NEXT:          "typename T"
 // CHECK-NEXT:        ]
@@ -62,15 +62,15 @@ template<Incrementable T> Incrementable auto incrementTwo(T t);
 // CHECK-NEXT:        "USR": "0000000000000000000000000000000000000000"
 // CHECK-NEXT:      },
 // CHECK-NEXT:      "Template": {
-// CHECK-NOT:         "Constraints": [
-// CHECK-NOT:           {
-// CHECK-NOT:             "Expression": "Incrementable<T>",
-// CHECK-NOT:             "Name": "Incrementable",
-// CHECK-NOT:             "Path": "",
-// CHECK-NOT:             "QualName": "Incrementable",
-// CHECK-NOT:             "USR": "{{[0-9A-F]*}}"
-// CHECK-NOT:           }
-// CHECK-NOT:         ],
+// CHECK-NEXT:        "Constraints": [
+// CHECK-NEXT:          {
+// CHECK-NEXT:            "Expression": "Incrementable<T>",
+// CHECK-NEXT:            "Name": "Incrementable",
+// CHECK-NEXT:            "Path": "",
+// CHECK-NEXT:            "QualName": "Incrementable",
+// CHECK-NEXT:            "USR": "{{[0-9A-F]*}}"
+// CHECK-NEXT:          }
+// CHECK-NEXT:        ],
 // CHECK-NEXT:        "Parameters": [
 // CHECK-NEXT:          "Incrementable T"
 // CHECK-NEXT:        ]
diff --git a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
index 659870d2a5c0..a38dfd303660 100644
--- a/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
+++ b/clang-tools-extra/unittests/clang-doc/BitcodeTest.cpp
@@ -37,6 +37,8 @@ static std::string writeInfo(Info *I) {
     return writeInfo(*static_cast<FunctionInfo *>(I));
   case InfoType::IT_typedef:
     return writeInfo(*static_cast<TypedefInfo *>(I));
+  case InfoType::IT_concept:
+    return writeInfo(*static_cast<ConceptInfo *>(I));
   case InfoType::IT_default:
     return "";
   }
-- 
cgit v1.2.3


From 72979093e79ca905eac7ce951423f8b0f81c28a8 Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <95053726+gandhi56@users.noreply.github.com>
Date: Fri, 20 Jun 2025 22:51:23 -0400
Subject: Revert "[Reland][InstCombine] Iterative replacement in PtrReplacer"
 (#145137)

Reverts llvm/llvm-project#144626
---
 .../InstCombine/InstCombineLoadStoreAlloca.cpp     | 165 +++++++++------------
 .../InstCombine/AMDGPU/ptr-replace-alloca.ll       |  79 ----------
 2 files changed, 69 insertions(+), 175 deletions(-)
 delete mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll

diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index 9aec90120d8b..a9751ab03e20 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -243,10 +243,11 @@ public:
   void replacePointer(Value *V);
 
 private:
+  bool collectUsersRecursive(Instruction &I);
   void replace(Instruction *I);
-  Value *getReplacement(Value *V) const { return WorkMap.lookup(V); }
+  Value *getReplacement(Value *I);
   bool isAvailable(Instruction *I) const {
-    return I == &Root || UsersToReplace.contains(I);
+    return I == &Root || Worklist.contains(I);
   }
 
   bool isEqualOrValidAddrSpaceCast(const Instruction *I,
@@ -258,7 +259,8 @@ private:
     return (FromAS == ToAS) || IC.isValidAddrSpaceCast(FromAS, ToAS);
   }
 
-  SmallSetVector<Instruction *, 32> UsersToReplace;
+  SmallPtrSet<Instruction *, 32> ValuesToRevisit;
+  SmallSetVector<Instruction *, 4> Worklist;
   MapVector<Value *, Value *> WorkMap;
   InstCombinerImpl &IC;
   Instruction &Root;
@@ -267,79 +269,72 @@ private:
 } // end anonymous namespace
 
 bool PointerReplacer::collectUsers() {
-  SmallVector<Instruction *> Worklist;
-  SmallSetVector<Instruction *, 32> ValuesToRevisit;
-
-  auto PushUsersToWorklist = [&](Instruction *Inst) {
-    for (auto *U : Inst->users())
-      if (auto *I = dyn_cast<Instruction>(U))
-        if (!isAvailable(I) && !ValuesToRevisit.contains(I))
-          Worklist.emplace_back(I);
-  };
+  if (!collectUsersRecursive(Root))
+    return false;
 
-  PushUsersToWorklist(&Root);
-  while (!Worklist.empty()) {
-    Instruction *Inst = Worklist.pop_back_val();
+  // Ensure that all outstanding (indirect) users of I
+  // are inserted into the Worklist. Return false
+  // otherwise.
+  return llvm::set_is_subset(ValuesToRevisit, Worklist);
+}
+
+bool PointerReplacer::collectUsersRecursive(Instruction &I) {
+  for (auto *U : I.users()) {
+    auto *Inst = cast<Instruction>(&*U);
     if (auto *Load = dyn_cast<LoadInst>(Inst)) {
       if (Load->isVolatile())
         return false;
-      UsersToReplace.insert(Load);
+      Worklist.insert(Load);
     } else if (auto *PHI = dyn_cast<PHINode>(Inst)) {
-      /// TODO: Handle poison and null pointers for PHI and select.
-      // If all incoming values are available, mark this PHI as
-      // replacable and push it's users into the worklist.
-      bool IsReplacable = true;
-      if (all_of(PHI->incoming_values(), [&](Value *V) {
-            if (!isa<Instruction>(V))
-              return IsReplacable = false;
-            return isAvailable(cast<Instruction>(V));
+      // All incoming values must be instructions for replacability
+      if (any_of(PHI->incoming_values(),
+                 [](Value *V) { return !isa<Instruction>(V); }))
+        return false;
+
+      // If at least one incoming value of the PHI is not in Worklist,
+      // store the PHI for revisiting and skip this iteration of the
+      // loop.
+      if (any_of(PHI->incoming_values(), [this](Value *V) {
+            return !isAvailable(cast<Instruction>(V));
           })) {
-        UsersToReplace.insert(PHI);
-        PushUsersToWorklist(PHI);
+        ValuesToRevisit.insert(Inst);
         continue;
       }
 
-      // Either an incoming value is not an instruction or not all
-      // incoming values are available. If this PHI was already
-      // visited prior to this iteration, return false.
-      if (!IsReplacable || !ValuesToRevisit.insert(PHI))
+      Worklist.insert(PHI);
+      if (!collectUsersRecursive(*PHI))
         return false;
-
-      // Push PHI back into the stack, followed by unavailable
-      // incoming values.
-      Worklist.emplace_back(PHI);
-      for (unsigned Idx = 0; Idx < PHI->getNumIncomingValues(); ++Idx) {
-        auto *IncomingValue = cast<Instruction>(PHI->getIncomingValue(Idx));
-        if (UsersToReplace.contains(IncomingValue))
-          continue;
-        if (!ValuesToRevisit.insert(IncomingValue))
-          return false;
-        Worklist.emplace_back(IncomingValue);
-      }
     } else if (auto *SI = dyn_cast<SelectInst>(Inst)) {
-      auto *TrueInst = dyn_cast<Instruction>(SI->getTrueValue());
-      auto *FalseInst = dyn_cast<Instruction>(SI->getFalseValue());
-      if (!TrueInst || !FalseInst)
+      if (!isa<Instruction>(SI->getTrueValue()) ||
+          !isa<Instruction>(SI->getFalseValue()))
         return false;
 
-      UsersToReplace.insert(SI);
-      PushUsersToWorklist(SI);
-    } else if (auto *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
-      UsersToReplace.insert(GEP);
-      PushUsersToWorklist(GEP);
+      if (!isAvailable(cast<Instruction>(SI->getTrueValue())) ||
+          !isAvailable(cast<Instruction>(SI->getFalseValue()))) {
+        ValuesToRevisit.insert(Inst);
+        continue;
+      }
+      Worklist.insert(SI);
+      if (!collectUsersRecursive(*SI))
+        return false;
+    } else if (isa<GetElementPtrInst>(Inst)) {
+      Worklist.insert(Inst);
+      if (!collectUsersRecursive(*Inst))
+        return false;
     } else if (auto *MI = dyn_cast<MemTransferInst>(Inst)) {
       if (MI->isVolatile())
         return false;
-      UsersToReplace.insert(Inst);
+      Worklist.insert(Inst);
     } else if (isEqualOrValidAddrSpaceCast(Inst, FromAS)) {
-      UsersToReplace.insert(Inst);
-      PushUsersToWorklist(Inst);
+      Worklist.insert(Inst);
+      if (!collectUsersRecursive(*Inst))
+        return false;
     } else if (Inst->isLifetimeStartOrEnd()) {
       continue;
     } else {
       // TODO: For arbitrary uses with address space mismatches, should we check
       // if we can introduce a valid addrspacecast?
-      LLVM_DEBUG(dbgs() << "Cannot handle pointer user: " << *Inst << '\n');
+      LLVM_DEBUG(dbgs() << "Cannot handle pointer user: " << *U << '\n');
       return false;
     }
   }
@@ -347,39 +342,7 @@ bool PointerReplacer::collectUsers() {
   return true;
 }
 
-void PointerReplacer::replacePointer(Value *V) {
-  assert(cast<PointerType>(Root.getType()) != cast<PointerType>(V->getType()) &&
-         "Invalid usage");
-  WorkMap[&Root] = V;
-  SmallVector<Instruction *> Worklist;
-  SetVector<Instruction *> PostOrderWorklist;
-  SmallPtrSet<Instruction *, 32> Visited;
-
-  // Perform a postorder traversal of the users of Root.
-  Worklist.push_back(&Root);
-  while (!Worklist.empty()) {
-    Instruction *I = Worklist.back();
-
-    // If I has not been processed before, push each of its
-    // replacable users into the worklist.
-    if (Visited.insert(I).second) {
-      for (auto *U : I->users()) {
-        auto *UserInst = cast<Instruction>(U);
-        if (UsersToReplace.contains(UserInst))
-          Worklist.push_back(UserInst);
-      }
-      // Otherwise, users of I have already been pushed into
-      // the PostOrderWorklist. Push I as well.
-    } else {
-      PostOrderWorklist.insert(I);
-      Worklist.pop_back();
-    }
-  }
-
-  // Replace pointers in reverse-postorder.
-  for (Instruction *I : reverse(PostOrderWorklist))
-    replace(I);
-}
+Value *PointerReplacer::getReplacement(Value *V) { return WorkMap.lookup(V); }
 
 void PointerReplacer::replace(Instruction *I) {
   if (getReplacement(I))
@@ -401,15 +364,13 @@ void PointerReplacer::replace(Instruction *I) {
     // replacement (new value).
     WorkMap[NewI] = NewI;
   } else if (auto *PHI = dyn_cast<PHINode>(I)) {
-    // Create a new PHI by replacing any incoming value that is a user of the
-    // root pointer and has a replacement.
-    Value *V = WorkMap.lookup(PHI->getIncomingValue(0));
-    PHI->mutateType(V ? V->getType() : PHI->getIncomingValue(0)->getType());
-    for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I) {
-      Value *V = WorkMap.lookup(PHI->getIncomingValue(I));
-      PHI->setIncomingValue(I, V ? V : PHI->getIncomingValue(I));
-    }
-    WorkMap[PHI] = PHI;
+    Type *NewTy = getReplacement(PHI->getIncomingValue(0))->getType();
+    auto *NewPHI = PHINode::Create(NewTy, PHI->getNumIncomingValues(),
+                                   PHI->getName(), PHI->getIterator());
+    for (unsigned int I = 0; I < PHI->getNumIncomingValues(); ++I)
+      NewPHI->addIncoming(getReplacement(PHI->getIncomingValue(I)),
+                          PHI->getIncomingBlock(I));
+    WorkMap[PHI] = NewPHI;
   } else if (auto *GEP = dyn_cast<GetElementPtrInst>(I)) {
     auto *V = getReplacement(GEP->getPointerOperand());
     assert(V && "Operand not replaced");
@@ -473,6 +434,18 @@ void PointerReplacer::replace(Instruction *I) {
   }
 }
 
+void PointerReplacer::replacePointer(Value *V) {
+#ifndef NDEBUG
+  auto *PT = cast<PointerType>(Root.getType());
+  auto *NT = cast<PointerType>(V->getType());
+  assert(PT != NT && "Invalid usage");
+#endif
+  WorkMap[&Root] = V;
+
+  for (Instruction *Workitem : Worklist)
+    replace(Workitem);
+}
+
 Instruction *InstCombinerImpl::visitAllocaInst(AllocaInst &AI) {
   if (auto *I = simplifyAllocaArraySize(*this, AI, DT))
     return I;
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll b/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll
deleted file mode 100644
index 538cc19f9722..000000000000
--- a/llvm/test/Transforms/InstCombine/AMDGPU/ptr-replace-alloca.ll
+++ /dev/null
@@ -1,79 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=instcombine -S < %s | FileCheck %s
-
-%struct.type = type { [256 x <2 x i64>] }
-@g1 = external hidden addrspace(3) global %struct.type, align 16
-
-; This test requires the PtrReplacer to replace users in an RPO traversal.
-; Furthermore, %ptr.else need not to be replaced so it must be retained in
-; %ptr.sink.
-define <2 x i64> @func(ptr addrspace(4) byref(%struct.type) align 16 %0, i1 %cmp.0) {
-; CHECK-LABEL: define <2 x i64> @func(
-; CHECK-SAME: ptr addrspace(4) byref([[STRUCT_TYPE:%.*]]) align 16 [[TMP0:%.*]], i1 [[CMP_0:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*:]]
-; CHECK-NEXT:    br i1 [[CMP_0]], label %[[IF_THEN:.*]], label %[[IF_ELSE:.*]]
-; CHECK:       [[IF_THEN]]:
-; CHECK-NEXT:    [[VAL_THEN:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr
-; CHECK-NEXT:    br label %[[SINK:.*]]
-; CHECK:       [[IF_ELSE]]:
-; CHECK-NEXT:    [[PTR_ELSE:%.*]] = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
-; CHECK-NEXT:    br label %[[SINK]]
-; CHECK:       [[SINK]]:
-; CHECK-NEXT:    [[PTR_SINK:%.*]] = phi ptr [ [[PTR_ELSE]], %[[IF_ELSE]] ], [ [[VAL_THEN]], %[[IF_THEN]] ]
-; CHECK-NEXT:    [[VAL_SINK:%.*]] = load <2 x i64>, ptr [[PTR_SINK]], align 16
-; CHECK-NEXT:    ret <2 x i64> [[VAL_SINK]]
-;
-entry:
-  %coerce = alloca %struct.type, align 16, addrspace(5)
-  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 16 %coerce, ptr addrspace(4) align 16 %0, i64 4096, i1 false)
-  br i1 %cmp.0, label %if.then, label %if.else
-
-if.then:                                    ; preds = %entry
-  %ptr.then = getelementptr inbounds i8, ptr addrspace(5) %coerce, i64 0
-  %val.then = addrspacecast ptr addrspace(5) %ptr.then to ptr
-  br label %sink
-
-if.else:                                      ; preds = %entry
-  %ptr.else = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
-  %val.else = getelementptr inbounds nuw i8, ptr %ptr.else, i64 0
-  br label %sink
-
-sink:
-  %ptr.sink = phi ptr [ %val.else, %if.else ], [ %val.then, %if.then ]
-  %val.sink = load <2 x i64>, ptr %ptr.sink, align 16
-  ret <2 x i64> %val.sink
-}
-
-define <2 x i64> @func_phi_loop(ptr addrspace(4) byref(%struct.type) align 16 %0, i1 %cmp.0) {
-; CHECK-LABEL: define <2 x i64> @func_phi_loop(
-; CHECK-SAME: ptr addrspace(4) byref([[STRUCT_TYPE:%.*]]) align 16 [[TMP0:%.*]], i1 [[CMP_0:%.*]]) {
-; CHECK-NEXT:  [[ENTRY:.*]]:
-; CHECK-NEXT:    [[VAL_0:%.*]] = addrspacecast ptr addrspace(4) [[TMP0]] to ptr
-; CHECK-NEXT:    br label %[[LOOP:.*]]
-; CHECK:       [[LOOP]]:
-; CHECK-NEXT:    [[PTR_PHI_R:%.*]] = phi ptr [ [[PTR_1:%.*]], %[[LOOP]] ], [ [[VAL_0]], %[[ENTRY]] ]
-; CHECK-NEXT:    [[PTR_1]] = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
-; CHECK-NEXT:    br i1 [[CMP_0]], label %[[LOOP]], label %[[SINK:.*]]
-; CHECK:       [[SINK]]:
-; CHECK-NEXT:    [[VAL_SINK:%.*]] = load <2 x i64>, ptr [[PTR_PHI_R]], align 16
-; CHECK-NEXT:    ret <2 x i64> [[VAL_SINK]]
-;
-entry:
-  %coerce = alloca %struct.type, align 16, addrspace(5)
-  call void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) align 16 %coerce, ptr addrspace(4) align 16 %0, i64 4096, i1 false)
-  %ptr.0 = getelementptr inbounds i8, ptr addrspace(5) %coerce, i64 0
-  %val.0 = addrspacecast ptr addrspace(5) %ptr.0 to ptr
-  br label %loop
-
-loop:
-  %ptr.phi = phi ptr [ %val.1, %loop ], [ %val.0, %entry ]
-  %ptr.1 = load ptr, ptr addrspace(3) getelementptr inbounds nuw (i8, ptr addrspace(3) @g1, i32 32), align 16
-  %val.1 = getelementptr inbounds nuw i8, ptr %ptr.1, i64 0
-  br i1 %cmp.0, label %loop, label %sink
-
-sink:
-  %val.sink = load <2 x i64>, ptr %ptr.phi, align 16
-  ret <2 x i64> %val.sink
-}
-
-declare void @llvm.memcpy.p5.p4.i64(ptr addrspace(5) noalias writeonly captures(none), ptr addrspace(4) noalias readonly captures(none), i64, i1 immarg) #0
-- 
cgit v1.2.3


From 17e8465a3eb0cae48b9f62d27fd26f2b070f1f9b Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Fri, 20 Jun 2025 20:06:32 -0700
Subject: AArch64: Replace AArch64MCExpr with MCSpecifierExpr

Replace AArch64MCExpr, which encodes expressions with relocation
specifiers, with the new generic MCSpecifierExpr interface, aligning
with other targets by phasing out target-specific XXXMCExpr classes.

Temporarily convert AArch64MCExpr to a namespace to avoid renaming
`AArch64MCExpr::VK_` constants in this PR. A follow-up patch will rename
these to `AArch64::S_` to match the convention used by other targets.

Move helper functions to AArch64MCAsmInfo.h, with the goal of eventually
removing AArch64MCExpr.h.

Pull Request: https://github.com/llvm/llvm-project/pull/144632
---
 bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp   | 47 +++++++++------------
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp      |  4 +-
 llvm/lib/Target/AArch64/AArch64MCInstLower.cpp     |  6 +--
 .../Target/AArch64/AsmParser/AArch64AsmParser.cpp  | 12 +++---
 .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp     | 26 ++++++------
 .../MCTargetDesc/AArch64ELFObjectWriter.cpp        |  9 ++--
 .../Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h | 12 ++++++
 .../AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp  |  4 +-
 .../Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp  | 10 -----
 .../Target/AArch64/MCTargetDesc/AArch64MCExpr.h    | 48 +++++++---------------
 .../MCTargetDesc/AArch64WinCOFFObjectWriter.cpp    | 10 ++---
 11 files changed, 81 insertions(+), 107 deletions(-)

diff --git a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
index eb1d9d8a1951..e6e0aeba3457 100644
--- a/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
+++ b/bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp
@@ -14,7 +14,7 @@
 #include "AArch64MCSymbolizer.h"
 #include "MCTargetDesc/AArch64AddressingModes.h"
 #include "MCTargetDesc/AArch64FixupKinds.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "bolt/Core/BinaryBasicBlock.h"
@@ -179,13 +179,10 @@ public:
 
   bool equals(const MCSpecifierExpr &A, const MCSpecifierExpr &B,
               CompFuncTy Comp) const override {
-    const auto &AArch64ExprA = cast<AArch64MCExpr>(A);
-    const auto &AArch64ExprB = cast<AArch64MCExpr>(B);
-    if (AArch64ExprA.getKind() != AArch64ExprB.getKind())
+    if (A.getSpecifier() != B.getSpecifier())
       return false;
 
-    return MCPlusBuilder::equals(*AArch64ExprA.getSubExpr(),
-                                 *AArch64ExprB.getSubExpr(), Comp);
+    return MCPlusBuilder::equals(*A.getSubExpr(), *B.getSubExpr(), Comp);
   }
 
   bool shortenInstruction(MCInst &, const MCSubtargetInfo &) const override {
@@ -1084,7 +1081,7 @@ public:
 
     if (isADR(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_LO21 ||
         RelType == ELF::R_AARCH64_TLSDESC_ADR_PREL21) {
-      return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx);
+      return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS, Ctx);
     } else if (isADRP(Inst) || RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21 ||
                RelType == ELF::R_AARCH64_ADR_PREL_PG_HI21_NC ||
                RelType == ELF::R_AARCH64_TLSDESC_ADR_PAGE21 ||
@@ -1092,7 +1089,7 @@ public:
                RelType == ELF::R_AARCH64_ADR_GOT_PAGE) {
       // Never emit a GOT reloc, we handled this in
       // RewriteInstance::readRelocations().
-      return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
+      return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, Ctx);
     } else {
       switch (RelType) {
       case ELF::R_AARCH64_ADD_ABS_LO12_NC:
@@ -1106,18 +1103,18 @@ public:
       case ELF::R_AARCH64_TLSDESC_LD64_LO12:
       case ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC:
       case ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC:
-        return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_LO12, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G3:
-        return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G3, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G3, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G2:
       case ELF::R_AARCH64_MOVW_UABS_G2_NC:
-        return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G2_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G2_NC, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G1:
       case ELF::R_AARCH64_MOVW_UABS_G1_NC:
-        return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G1_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G1_NC, Ctx);
       case ELF::R_AARCH64_MOVW_UABS_G0:
       case ELF::R_AARCH64_MOVW_UABS_G0_NC:
-        return AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_G0_NC, Ctx);
+        return MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_G0_NC, Ctx);
       default:
         break;
       }
@@ -1142,7 +1139,7 @@ public:
   }
 
   const MCSymbol *getTargetSymbol(const MCExpr *Expr) const override {
-    auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr);
+    auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Expr);
     if (AArchExpr && AArchExpr->getSubExpr())
       return getTargetSymbol(AArchExpr->getSubExpr());
 
@@ -1162,7 +1159,7 @@ public:
   }
 
   int64_t getTargetAddend(const MCExpr *Expr) const override {
-    auto *AArchExpr = dyn_cast<AArch64MCExpr>(Expr);
+    auto *AArchExpr = dyn_cast<MCSpecifierExpr>(Expr);
     if (AArchExpr && AArchExpr->getSubExpr())
       return getTargetAddend(AArchExpr->getSubExpr());
 
@@ -2030,9 +2027,8 @@ public:
     MCInst Inst;
     Inst.setOpcode(AArch64::MOVZXi);
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
-    Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        AArch64MCExpr::VK_ABS_G3, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(
+        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G3, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x30));
     Seq.emplace_back(Inst);
 
@@ -2040,9 +2036,8 @@ public:
     Inst.setOpcode(AArch64::MOVKXi);
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
-    Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        AArch64MCExpr::VK_ABS_G2_NC, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(
+        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G2_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x20));
     Seq.emplace_back(Inst);
 
@@ -2050,9 +2045,8 @@ public:
     Inst.setOpcode(AArch64::MOVKXi);
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
-    Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        AArch64MCExpr::VK_ABS_G1_NC, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(
+        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G1_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0x10));
     Seq.emplace_back(Inst);
 
@@ -2060,9 +2054,8 @@ public:
     Inst.setOpcode(AArch64::MOVKXi);
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
     Inst.addOperand(MCOperand::createReg(AArch64::X16));
-    Inst.addOperand(MCOperand::createExpr(AArch64MCExpr::create(
-        MCSymbolRefExpr::create(Target, MCSymbolRefExpr::VK_None, *Ctx),
-        AArch64MCExpr::VK_ABS_G0_NC, *Ctx)));
+    Inst.addOperand(MCOperand::createExpr(
+        MCSpecifierExpr::create(Target, AArch64MCExpr::VK_ABS_G0_NC, *Ctx)));
     Inst.addOperand(MCOperand::createImm(0));
     Seq.emplace_back(Inst);
 
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 4099f40ea07f..a16c104d8bef 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -910,13 +910,13 @@ void AArch64AsmPrinter::emitHwasanMemaccessSymbols(Module &M) {
       // have a chance to save them.
       EmitToStreamer(MCInstBuilder(AArch64::ADRP)
                          .addReg(AArch64::X16)
-                         .addExpr(AArch64MCExpr::create(
+                         .addExpr(MCSpecifierExpr::create(
                              HwasanTagMismatchRef, AArch64MCExpr::VK_GOT_PAGE,
                              OutContext)));
       EmitToStreamer(MCInstBuilder(AArch64::LDRXui)
                          .addReg(AArch64::X16)
                          .addReg(AArch64::X16)
-                         .addExpr(AArch64MCExpr::create(
+                         .addExpr(MCSpecifierExpr::create(
                              HwasanTagMismatchRef, AArch64MCExpr::VK_GOT_LO12,
                              OutContext)));
       EmitToStreamer(MCInstBuilder(AArch64::BR).addReg(AArch64::X16));
diff --git a/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp b/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
index eb699c75cf10..fd3ce6c72e50 100644
--- a/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MCInstLower.cpp
@@ -171,7 +171,7 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandMachO(const MachineOperand &MO,
              AArch64II::MO_PAGEOFF)
       Spec = AArch64MCExpr::M_PAGEOFF;
   }
-  // TODO: Migrate to AArch64MCExpr::create like ELF.
+  // TODO: Migrate to MCSpecifierExpr::create like ELF.
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, Spec, Ctx);
   if (!MO.isJTI() && MO.getOffset())
     Expr = MCBinaryExpr::createAdd(
@@ -265,7 +265,7 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO,
 
   AArch64MCExpr::Specifier RefKind;
   RefKind = static_cast<AArch64MCExpr::Specifier>(RefFlags);
-  Expr = AArch64MCExpr::create(Expr, RefKind, Ctx);
+  Expr = MCSpecifierExpr::create(Expr, RefKind, Ctx);
 
   return MCOperand::createExpr(Expr);
 }
@@ -320,7 +320,7 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandCOFF(const MachineOperand &MO,
   auto RefKind = static_cast<AArch64MCExpr::Specifier>(RefFlags);
   assert(RefKind != AArch64MCExpr::VK_INVALID &&
          "Invalid relocation requested");
-  Expr = AArch64MCExpr::create(Expr, RefKind, Ctx);
+  Expr = MCSpecifierExpr::create(Expr, RefKind, Ctx);
 
   return MCOperand::createExpr(Expr);
 }
diff --git a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
index d8bdc01a3454..faa82abbd898 100644
--- a/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
+++ b/llvm/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp
@@ -3307,8 +3307,8 @@ ParseStatus AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) {
         ELFSpec == AArch64MCExpr::VK_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADRP relocation (unfortunately).
-      Expr =
-          AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE, getContext());
+      Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS_PAGE,
+                                     getContext());
     } else if ((DarwinSpec == AArch64MCExpr::M_GOTPAGE ||
                 DarwinSpec == AArch64MCExpr::M_TLVPPAGE) &&
                Addend != 0) {
@@ -3361,7 +3361,7 @@ ParseStatus AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) {
         ELFSpec == AArch64MCExpr::VK_INVALID) {
       // No modifier was specified at all; this is the syntax for an ELF basic
       // ADR relocation (unfortunately).
-      Expr = AArch64MCExpr::create(Expr, AArch64MCExpr::VK_ABS, getContext());
+      Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_ABS, getContext());
     } else if (ELFSpec != AArch64MCExpr::VK_GOT_AUTH_PAGE) {
       // For tiny code model, we use :got_auth: operator to fill 21-bit imm of
       // adr. It's not actually GOT entry page address but the GOT address
@@ -4478,7 +4478,7 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) {
     return true;
 
   if (HasELFModifier)
-    ImmVal = AArch64MCExpr::create(ImmVal, RefKind, getContext());
+    ImmVal = MCSpecifierExpr::create(ImmVal, RefKind, getContext());
 
   SMLoc EndLoc;
   if (getContext().getAsmInfo()->hasSubsectionsViaSymbols()) {
@@ -7360,7 +7360,7 @@ bool AArch64AsmParser::parseDirectiveTLSDescCall(SMLoc L) {
 
   MCSymbol *Sym = getContext().getOrCreateSymbol(Name);
   const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext());
-  Expr = AArch64MCExpr::create(Expr, AArch64MCExpr::VK_TLSDESC, getContext());
+  Expr = MCSpecifierExpr::create(Expr, AArch64MCExpr::VK_TLSDESC, getContext());
 
   MCInst Inst;
   Inst.setOpcode(AArch64::TLSDESCCALL);
@@ -8288,7 +8288,7 @@ bool AArch64AsmParser::classifySymbolRef(const MCExpr *Expr,
   DarwinSpec = AArch64MCExpr::None;
   Addend = 0;
 
-  if (const AArch64MCExpr *AE = dyn_cast<AArch64MCExpr>(Expr)) {
+  if (auto *AE = dyn_cast<MCSpecifierExpr>(Expr)) {
     ELFSpec = AE->getSpecifier();
     Expr = AE->getSubExpr();
   }
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
index 0d29316d843e..88ba2ef3fe1f 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp
@@ -7,7 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/AArch64FixupKinds.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "Utils/AArch64BaseInfo.h"
 #include "llvm/BinaryFormat/MachO.h"
@@ -221,8 +221,8 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
   case AArch64::fixup_aarch64_movw: {
     AArch64MCExpr::Specifier RefKind =
         static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-    if (AArch64MCExpr::getSymbolLoc(RefKind) != AArch64MCExpr::VK_ABS &&
-        AArch64MCExpr::getSymbolLoc(RefKind) != AArch64MCExpr::VK_SABS) {
+    if (AArch64::getSymbolLoc(RefKind) != AArch64MCExpr::VK_ABS &&
+        AArch64::getSymbolLoc(RefKind) != AArch64MCExpr::VK_SABS) {
       if (!RefKind) {
         // The fixup is an expression
         if (SignedValue > 0xFFFF || SignedValue < -0xFFFF)
@@ -250,8 +250,8 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       return Value;
     }
 
-    if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
-      switch (AArch64MCExpr::getAddressFrag(RefKind)) {
+    if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
+      switch (AArch64::getAddressFrag(RefKind)) {
       case AArch64MCExpr::VK_G0:
         break;
       case AArch64MCExpr::VK_G1:
@@ -268,7 +268,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       }
 
     } else {
-      switch (AArch64MCExpr::getAddressFrag(RefKind)) {
+      switch (AArch64::getAddressFrag(RefKind)) {
       case AArch64MCExpr::VK_G0:
         break;
       case AArch64MCExpr::VK_G1:
@@ -287,8 +287,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
 
     if (RefKind & AArch64MCExpr::VK_NC) {
       Value &= 0xFFFF;
-    }
-    else if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
+    } else if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS) {
       if (SignedValue > 0xFFFF || SignedValue < -0xFFFF)
         Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
 
@@ -296,8 +295,7 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, const MCValue &Target,
       if (SignedValue < 0)
         SignedValue = ~SignedValue;
       Value = static_cast<uint64_t>(SignedValue);
-    }
-    else if (Value > 0xFFFF) {
+    } else if (Value > 0xFFFF) {
       Ctx.reportError(Fixup.getLoc(), "fixup value out of range");
     }
     return Value;
@@ -424,9 +422,9 @@ void AArch64AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
 
   if (Fixup.getTargetKind() == FK_Data_8 && TheTriple.isOSBinFormatELF()) {
     auto RefKind = static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-    AArch64MCExpr::Specifier SymLoc = AArch64MCExpr::getSymbolLoc(RefKind);
-    if (SymLoc == AArch64AuthMCExpr::VK_AUTH ||
-        SymLoc == AArch64AuthMCExpr::VK_AUTHADDR) {
+    AArch64MCExpr::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
+    if (SymLoc == AArch64MCExpr::VK_AUTH ||
+        SymLoc == AArch64MCExpr::VK_AUTHADDR) {
       const auto *Expr = dyn_cast<AArch64AuthMCExpr>(Fixup.getValue());
       if (!Expr) {
         getContext().reportError(Fixup.getValue()->getLoc(),
@@ -479,7 +477,7 @@ void AArch64AsmBackend::applyFixup(const MCFragment &, const MCFixup &Fixup,
   // handle this more cleanly. This may affect the output of -show-mc-encoding.
   AArch64MCExpr::Specifier RefKind =
       static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-  if (AArch64MCExpr::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS ||
+  if (AArch64::getSymbolLoc(RefKind) == AArch64MCExpr::VK_SABS ||
       (!RefKind && Fixup.getTargetKind() == AArch64::fixup_aarch64_movw)) {
     // If the immediate is negative, generate MOVN else MOVZ.
     // (Bit 30 = 0) ==> MOVN, (Bit 30 = 1) ==> MOVZ.
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
index df2e13a35dcb..28aa4644a55f 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp
@@ -12,7 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/AArch64FixupKinds.h"
-#include "MCTargetDesc/AArch64MCExpr.h"
+#include "MCTargetDesc/AArch64MCAsmInfo.h"
 #include "MCTargetDesc/AArch64MCTargetDesc.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/MC/MCContext.h"
@@ -88,8 +88,8 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
   unsigned Kind = Fixup.getTargetKind();
   AArch64MCExpr::Specifier RefKind =
       static_cast<AArch64MCExpr::Specifier>(Target.getSpecifier());
-  AArch64MCExpr::Specifier SymLoc = AArch64MCExpr::getSymbolLoc(RefKind);
-  bool IsNC = AArch64MCExpr::isNotChecked(RefKind);
+  AArch64MCExpr::Specifier SymLoc = AArch64::getSymbolLoc(RefKind);
+  bool IsNC = AArch64::isNotChecked(RefKind);
 
   switch (SymLoc) {
   case AArch64MCExpr::VK_DTPREL:
@@ -356,8 +356,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(const MCFixup &Fixup,
       if ((SymLoc == AArch64MCExpr::VK_GOT ||
            SymLoc == AArch64MCExpr::VK_GOT_AUTH) &&
           IsNC) {
-        AArch64MCExpr::Specifier AddressLoc =
-            AArch64MCExpr::getAddressFrag(RefKind);
+        AArch64MCExpr::Specifier AddressLoc = AArch64::getAddressFrag(RefKind);
         bool IsAuth = (SymLoc == AArch64MCExpr::VK_GOT_AUTH);
         if (!IsILP32) {
           if (AddressLoc == AArch64MCExpr::VK_LO15)
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
index bc02586d7388..58e87f8a208f 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h
@@ -58,9 +58,21 @@ struct AArch64MCAsmInfoGNUCOFF : public MCAsmInfoGNUCOFF {
 };
 
 namespace AArch64 {
+using Specifier = uint16_t;
+
 /// Return the string representation of the ELF relocation specifier
 /// (e.g. ":got:", ":lo12:").
 StringRef getSpecifierName(const MCSpecifierExpr &Expr);
+
+inline Specifier getSymbolLoc(Specifier S) {
+  return static_cast<Specifier>(S & AArch64MCExpr::VK_SymLocBits);
+}
+
+inline Specifier getAddressFrag(Specifier S) {
+  return static_cast<Specifier>(S & AArch64MCExpr::VK_AddressFragBits);
+}
+
+inline bool isNotChecked(Specifier S) { return S & AArch64MCExpr::VK_NC; }
 } // namespace AArch64
 
 } // namespace llvm
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
index 4dc30b48c902..6db0d7de45e5 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp
@@ -308,7 +308,7 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx,
 
   // Set the shift bit of the add instruction for relocation types
   // R_AARCH64_TLSLE_ADD_TPREL_HI12 and R_AARCH64_TLSLD_ADD_DTPREL_HI12.
-  if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(Expr)) {
+  if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
     AArch64MCExpr::Specifier RefKind = A64E->getSpecifier();
     if (RefKind == AArch64MCExpr::VK_TPREL_HI12 ||
         RefKind == AArch64MCExpr::VK_DTPREL_HI12 ||
@@ -718,7 +718,7 @@ unsigned AArch64MCCodeEmitter::fixMOVZ(const MCInst &MI, unsigned EncodedValue,
     return EncodedValue;
 
   const MCExpr *E = UImm16MO.getExpr();
-  if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(E)) {
+  if (auto *A64E = dyn_cast<MCSpecifierExpr>(E)) {
     switch (A64E->getSpecifier()) {
     case AArch64MCExpr::VK_DTPREL_G2:
     case AArch64MCExpr::VK_DTPREL_G1:
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
index 7a7c6f7effd9..a3f58ca4ee14 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp
@@ -5,11 +5,6 @@
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
-//
-// This file contains the implementation of the relocation specifiers
-// accepted by the AArch64 architecture (e.g. ":lo12:", ":gottprel_g1:", ...).
-//
-//===----------------------------------------------------------------------===//
 
 #include "AArch64MCExpr.h"
 #include "AArch64MCAsmInfo.h"
@@ -20,11 +15,6 @@
 
 using namespace llvm;
 
-const AArch64MCExpr *AArch64MCExpr::create(const MCExpr *Expr, Specifier S,
-                                           MCContext &Ctx) {
-  return new (Ctx) AArch64MCExpr(Expr, S);
-}
-
 const AArch64AuthMCExpr *AArch64AuthMCExpr::create(const MCExpr *Expr,
                                                    uint16_t Discriminator,
                                                    AArch64PACKey::ID Key,
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
index 541f24c943a1..8ffd14e338e1 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h
@@ -21,11 +21,10 @@
 
 namespace llvm {
 
-class AArch64MCExpr : public MCSpecifierExpr {
-public:
-  using Specifier = uint16_t;
-  enum {
-    // clang-format off
+namespace AArch64MCExpr {
+using Specifier = uint16_t;
+enum {
+  // clang-format off
     None          = 0,
     // Symbol locations specifying (roughly speaking) what calculation should be
     // performed to construct the final address for the relocated
@@ -137,35 +136,18 @@ public:
     M_TLVPPAGEOFF,
 
     VK_INVALID  = 0xfff
-    // clang-format on
-  };
-
-protected:
-  explicit AArch64MCExpr(const MCExpr *Expr, Specifier S)
-      : MCSpecifierExpr(Expr, S) {}
-
-public:
-  static const AArch64MCExpr *create(const MCExpr *Expr, Specifier,
-                                     MCContext &Ctx);
-
-  static Specifier getSymbolLoc(Specifier S) {
-    return static_cast<Specifier>(S & VK_SymLocBits);
-  }
-
-  static Specifier getAddressFrag(Specifier S) {
-    return static_cast<Specifier>(S & VK_AddressFragBits);
-  }
-
-  static bool isNotChecked(Specifier S) { return S & VK_NC; }
+  // clang-format on
 };
+} // namespace AArch64MCExpr
 
-class AArch64AuthMCExpr final : public AArch64MCExpr {
+class AArch64AuthMCExpr final : public MCSpecifierExpr {
   uint16_t Discriminator;
   AArch64PACKey::ID Key;
 
   explicit AArch64AuthMCExpr(const MCExpr *Expr, uint16_t Discriminator,
                              AArch64PACKey::ID Key, bool HasAddressDiversity)
-      : AArch64MCExpr(Expr, HasAddressDiversity ? VK_AUTHADDR : VK_AUTH),
+      : MCSpecifierExpr(Expr, HasAddressDiversity ? AArch64MCExpr::VK_AUTHADDR
+                                                  : AArch64MCExpr::VK_AUTH),
         Discriminator(Discriminator), Key(Key) {}
 
 public:
@@ -175,16 +157,16 @@ public:
 
   AArch64PACKey::ID getKey() const { return Key; }
   uint16_t getDiscriminator() const { return Discriminator; }
-  bool hasAddressDiversity() const { return getSpecifier() == VK_AUTHADDR; }
+  bool hasAddressDiversity() const {
+    return getSpecifier() == AArch64MCExpr::VK_AUTHADDR;
+  }
 
   void print(raw_ostream &OS, const MCAsmInfo *MAI) const;
 
   static bool classof(const MCExpr *E) {
-    return isa<AArch64MCExpr>(E) && classof(cast<AArch64MCExpr>(E));
-  }
-
-  static bool classof(const AArch64MCExpr *E) {
-    return E->getSpecifier() == VK_AUTH || E->getSpecifier() == VK_AUTHADDR;
+    auto *SE = dyn_cast<MCSpecifierExpr>(E);
+    return SE && (SE->getSpecifier() == AArch64MCExpr::VK_AUTH ||
+                  SE->getSpecifier() == AArch64MCExpr::VK_AUTHADDR);
   }
 };
 } // end namespace llvm
diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
index 2e997631655e..70df95029c33 100644
--- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64WinCOFFObjectWriter.cpp
@@ -64,9 +64,9 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
   auto Spec = Target.getSpecifier();
   const MCExpr *Expr = Fixup.getValue();
 
-  if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(Expr)) {
+  if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
     AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
-    switch (AArch64MCExpr::getSymbolLoc(Spec)) {
+    switch (AArch64::getSymbolLoc(Spec)) {
     case AArch64MCExpr::VK_ABS:
     case AArch64MCExpr::VK_SECREL:
       // Supported
@@ -81,7 +81,7 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
 
   switch (FixupKind) {
   default: {
-    if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(Expr)) {
+    if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
       Ctx.reportError(Fixup.getLoc(), "relocation specifier " +
                                           AArch64::getSpecifierName(*A64E) +
                                           " unsupported on COFF targets");
@@ -116,7 +116,7 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
     return COFF::IMAGE_REL_ARM64_SECREL;
 
   case AArch64::fixup_aarch64_add_imm12:
-    if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(Expr)) {
+    if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
       AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
       if (Spec == AArch64MCExpr::VK_SECREL_LO12)
         return COFF::IMAGE_REL_ARM64_SECREL_LOW12A;
@@ -130,7 +130,7 @@ unsigned AArch64WinCOFFObjectWriter::getRelocType(
   case AArch64::fixup_aarch64_ldst_imm12_scale4:
   case AArch64::fixup_aarch64_ldst_imm12_scale8:
   case AArch64::fixup_aarch64_ldst_imm12_scale16:
-    if (const AArch64MCExpr *A64E = dyn_cast<AArch64MCExpr>(Expr)) {
+    if (auto *A64E = dyn_cast<MCSpecifierExpr>(Expr)) {
       AArch64MCExpr::Specifier Spec = A64E->getSpecifier();
       if (Spec == AArch64MCExpr::VK_SECREL_LO12)
         return COFF::IMAGE_REL_ARM64_SECREL_LOW12L;
-- 
cgit v1.2.3